[CI] fixing ci, removing soft cts ci

allocation

.gitea/workflows/Build.yml

@@ -22,8 +22,8 @@ jobs:
         run: |
           set -euxo pipefail
 
-          apt-get update
+          apt update
-          apt-get install -y unzip curl
+          apt install -y unzip curl
 
           MPSS_URL="https://drive.kbz8.me/dav/public-files/KSKeiOKxnFPyfws/mpss.zip"
           MPSS_ZIP="/tmp/mpss.zip"
@@ -52,8 +52,13 @@ jobs:
           echo "$PATH"
           ls -la /opt/mpss/3.8.6/sysroots/k1om-mpss-linux/bin | head -50
 
+      - name: Check formatting
+        run: zig fmt --check src/
+
       - name: ZLint pass
-        uses: DonIsaac/zlint-action@v0.0.1
+        run: |
+          curl -fsSL https://raw.githubusercontent.com/DonIsaac/zlint/refs/heads/main/tasks/install.sh | bash
+          zlint --deny-warnings
 
       - name: Building Ape
         run: zig build ape --release=safe

.gitea/workflows/CTS_Soft.yml

@@ -1,64 +0,0 @@
-name: CTS Soft
-
-on:
-  workflow_dispatch:
-
-jobs:
-  build_and_test:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-
-    - uses: https://codeberg.org/mlugg/setup-zig@v2
-    - name: Install system dependencies
-      run: |
-        apt update
-        apt install -y libgl1 libwayland-egl1 libwayland-cursor0 clang libwayland-server0
-
-    - name: Install Rust
-      uses: dtolnay/rust-toolchain@stable
-      with:
-        toolchain: stable
-        targets: x86_64-unknown-linux-gnu
-
-    - uses: Swatinem/rust-cache@v2
-      with:
-        cache-all-crates: "true"
-        cache-on-failure: "false"
-
-    - name: Install deqp-runner
-      run: cargo install deqp-runner
-
-    - name: Verify installations
-      shell: bash
-      run: |
-        source $HOME/.cargo/env
-        echo "Verifying tool installations..."
-        which deqp-runner && deqp-runner --version || echo "deqp-runner not found"
-
-    - name: Run Vulkan CTS
-      run: zig build cts-soft --release=fast -- -j4
-      continue-on-error: true
-
-    - name: Verify tests
-      run: ls cts | grep "results.csv";
-
-    - name: Archiving results
-      run: |
-        mkdir -p assets/
-        zip ./assets/results.zip ./cts/results.csv
-
-    - name: Deploying CTS results
-      uses: milanmk/actions-file-deployer@master
-      with:
-        remote-protocol: sftp
-        remote-host: ${{ secrets.SFTP_HOST_CTS_SOFT }}
-        remote-user: ${{ secrets.SFTP_USER_CTS_SOFT }}
-        remote-password: ${{ secrets.SFTP_PASSWORD_CTS_SOFT }}
-        remote-port: 6969
-        local-path: "./assets"
-        remote-path: "/www/assets"
-        sync: full

.gitea/workflows/Test.yml

@@ -19,8 +19,8 @@ jobs:
         run: |
           set -euxo pipefail
 
-          apt-get update
+          apt update
-          apt-get install -y unzip curl
+          apt install -y unzip curl
 
           MPSS_URL="https://drive.kbz8.me/dav/public-files/KSKeiOKxnFPyfws/mpss.zip"
           MPSS_ZIP="/tmp/mpss.zip"

README.md

@@ -129,8 +129,8 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCreateSwapchainKHR                             | ✅ Implemented |
 | vkCreateWaylandSurfaceKHR                        | ✅ Implemented |
 | vkCreateWin32SurfaceKHR                          | ⚙️ WIP         |
-| vkCreateXcbSurfaceKHR                            | ⚙️ WIP         |
+| vkCreateXcbSurfaceKHR                            | ✅ Implemented |
-| vkCreateXlibSurfaceKHR                           | ⚙️ WIP         |
+| vkCreateXlibSurfaceKHR                           | ✅ Implemented |
 | vkDestroyBuffer                                  | ✅ Implemented |
 | vkDestroyBufferView                              | ✅ Implemented |
 | vkDestroyCommandPool                             | ✅ Implemented |
@@ -194,8 +194,8 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkGetPhysicalDeviceSurfaceSupportKHR             | ✅ Implemented |
 | vkGetPhysicalDeviceWaylandPresentationSupportKHR | ✅ Implemented |
 | vkGetPhysicalDeviceWin32PresentationSupportKHR   | ⚙️ WIP         |
-| vkGetPhysicalDeviceXcbPresentationSupportKHR     | ⚙️ WIP         |
+| vkGetPhysicalDeviceXcbPresentationSupportKHR     | ✅ Implemented |
-| vkGetPhysicalDeviceXlibPresentationSupportKHR    | ⚙️ WIP         |
+| vkGetPhysicalDeviceXlibPresentationSupportKHR    | ✅ Implemented |
 | vkGetPipelineCacheData                           | ✅ Implemented |
 | vkGetQueryPoolResults                            | ✅ Implemented |
 | vkGetRenderAreaGranularity                       | ✅ Implemented |
@@ -235,7 +235,9 @@ To bring forth the driver:
 zig build phi --release=[fast|safe|small]
 ```
 
-And for those who seek another manner of building:
+In this manner, the daemon is cross-compiled by way of `k1om-mpss-linux-gcc`, delivered unto its destination via SSH, and speaketh through the SCIF.
+
+For those who seek a different method of creation:
 
 ```
 zig build --help
@@ -262,7 +264,7 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkAllocateDescriptorSets                         | ⚙️ WIP         |
 | vkAllocateMemory                                 | ✅ Implemented |
 | vkBeginCommandBuffer                             | ✅ Implemented |
-| vkBindBufferMemory                               | ⚙️ WIP         |
+| vkBindBufferMemory                               | ✅ Implemented |
 | vkBindImageMemory                                | ✅ Implemented |
 | vkCmdBeginQuery                                  | ⚙️ WIP         |
 | vkCmdBeginRenderPass                             | ⚙️ WIP         |
@@ -270,13 +272,13 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCmdBindIndexBuffer                             | ⚙️ WIP         |
 | vkCmdBindPipeline                                | ⚙️ WIP         |
 | vkCmdBindVertexBuffers                           | ⚙️ WIP         |
-| vkCmdBlitImage                                   | ✅ Implemented |
+| vkCmdBlitImage                                   | ⚙️ WIP         |
 | vkCmdClearAttachments                            | ⚙️ WIP         |
 | vkCmdClearColorImage                             | ⚙️ WIP         |
 | vkCmdClearDepthStencilImage                      | ⚙️ WIP         |
 | vkCmdCopyBuffer                                  | ✅ Implemented |
 | vkCmdCopyBufferToImage                           | ✅ Implemented |
-| vkCmdCopyImage                                   | ⚙️ WIP         |
+| vkCmdCopyImage                                   | ✅ Implemented |
 | vkCmdCopyImageToBuffer                           | ✅ Implemented |
 | vkCmdCopyQueryPoolResults                        | ⚙️ WIP         |
 | vkCmdDispatch                                    | ⚙️ WIP         |
@@ -310,7 +312,7 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCmdUpdateBuffer                                | ⚙️ WIP         |
 | vkCmdWaitEvents                                  | ⚙️ WIP         |
 | vkCmdWriteTimestamp                              | ⚙️ WIP         |
-| vkCreateBuffer                                   | ⚙️ WIP         |
+| vkCreateBuffer                                   | ✅ Implemented |
 | vkCreateBufferView                               | ⚙️ WIP         |
 | vkCreateCommandPool                              | ✅ Implemented |
 | vkCreateComputePipelines                         | ⚙️ WIP         |
@@ -318,7 +320,7 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCreateDescriptorSetLayout                      | ⚙️ WIP         |
 | vkCreateDevice                                   | ✅ Implemented |
 | vkCreateEvent                                    | ⚙️ WIP         |
-| vkCreateFence                                    | ⚙️ WIP         |
+| vkCreateFence                                    | ✅ Implemented |
 | vkCreateFramebuffer                              | ⚙️ WIP         |
 | vkCreateGraphicsPipelines                        | ⚙️ WIP         |
 | vkCreateImage                                    | ✅ Implemented |
@@ -330,20 +332,20 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCreateRenderPass                               | ⚙️ WIP         |
 | vkCreateSampler                                  | ⚙️ WIP         |
 | vkCreateSemaphore                                | ⚙️ WIP         |
-| vkCreateShaderModule                             | ⚙️ WIP         |
+| vkCreateShaderModule                             | ✅ Implemented |
-| vkCreateSwapchainKHR                             | ⚙️ WIP         |
+| vkCreateSwapchainKHR                             | ✅ Implemented |
-| vkCreateWaylandSurfaceKHR                        | ⚙️ WIP         |
+| vkCreateWaylandSurfaceKHR                        | ✅ Implemented |
 | vkCreateWin32SurfaceKHR                          | ⚙️ WIP         |
-| vkCreateXcbSurfaceKHR                            | ⚙️ WIP         |
+| vkCreateXcbSurfaceKHR                            | ✅ Implemented |
-| vkCreateXlibSurfaceKHR                           | ⚙️ WIP         |
+| vkCreateXlibSurfaceKHR                           | ✅ Implemented |
-| vkDestroyBuffer                                  | ⚙️ WIP         |
+| vkDestroyBuffer                                  | ✅ Implemented |
 | vkDestroyBufferView                              | ⚙️ WIP         |
 | vkDestroyCommandPool                             | ✅ Implemented |
 | vkDestroyDescriptorPool                          | ⚙️ WIP         |
 | vkDestroyDescriptorSetLayout                     | ⚙️ WIP         |
 | vkDestroyDevice                                  | ✅ Implemented |
 | vkDestroyEvent                                   | ⚙️ WIP         |
-| vkDestroyFence                                   | ⚙️ WIP         |
+| vkDestroyFence                                   | ✅ Implemented |
 | vkDestroyFramebuffer                             | ⚙️ WIP         |
 | vkDestroyImage                                   | ✅ Implemented |
 | vkDestroyImageView                               | ⚙️ WIP         |
@@ -354,11 +356,11 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkDestroyQueryPool                               | ⚙️ WIP         |
 | vkDestroyRenderPass                              | ⚙️ WIP         |
 | vkDestroySampler                                 | ⚙️ WIP         |
-| vkDestroySemaphore                               | ⚙️ WIP         |
+| vkDestroySemaphore                               | ✅ Implemented |
 | vkDestroyShaderModule                            | ⚙️ WIP         |
 | vkDestroySurfaceKHR                              | ⚙️ WIP         |
-| vkDestroySwapchainKHR                            | ⚙️ WIP         |
+| vkDestroySwapchainKHR                            | ✅ Implemented |
-| vkDeviceWaitIdle                                 | ⚙️ WIP         |
+| vkDeviceWaitIdle                                 | ✅ Implemented |
 | vkEndCommandBuffer                               | ✅ Implemented |
 | vkEnumerateDeviceExtensionProperties             | ⚙️ WIP         |
 | vkEnumerateDeviceLayerProperties                 | ⚙️ WIP         |
@@ -367,7 +369,7 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkEnumeratePhysicalDeviceGroupsKHR               | ⚙️ WIP         |
 | vkEnumeratePhysicalDevices                       | ✅ Implemented |
 | vkFlushMappedMemoryRanges                        | ⚙️ WIP         |
-| vkFreeCommandBuffers                             | ⚙️ WIP         |
+| vkFreeCommandBuffers                             | ✅ Implemented |
 | vkFreeDescriptorSets                             | ⚙️ WIP         |
 | vkFreeMemory                                     | ✅ Implemented |
 | vkGetBufferDeviceAddress                         | ⚙️ WIP         |
@@ -378,14 +380,14 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkGetDeviceGroupPresentCapabilitiesKHR           | ⚙️ WIP         |
 | vkGetDeviceGroupSurfacePresentModesKHR           | ⚙️ WIP         |
 | vkGetDeviceMemoryCommitment                      | ⚙️ WIP         |
-| vkGetDeviceProcAddr                              | ⚙️ WIP         |
+| vkGetDeviceProcAddr                              | ✅ Implemented |
-| vkGetDeviceQueue                                 | ⚙️ WIP         |
+| vkGetDeviceQueue                                 | ✅ Implemented |
-| vkGetEventStatus                                 | ⚙️ WIP         |
+| vkGetEventStatus                                 | ✅ Implemented |
-| vkGetFenceStatus                                 | ⚙️ WIP         |
+| vkGetFenceStatus                                 | ✅ Implemented |
 | vkGetImageMemoryRequirements                     | ✅ Implemented |
 | vkGetImageSparseMemoryRequirements               | ⚙️ WIP         |
 | vkGetImageSubresourceLayout                      | ✅ Implemented |
-| vkGetInstanceProcAddr                            | ⚙️ WIP         |
+| vkGetInstanceProcAddr                            | ✅ Implemented |
 | vkGetPhysicalDeviceFeatures                      | ✅ Implemented |
 | vkGetPhysicalDeviceFormatProperties              | ⚙️ WIP         |
 | vkGetPhysicalDeviceImageFormatProperties         | ✅ Implemented |
@@ -406,22 +408,22 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkGetRenderAreaGranularity                       | ⚙️ WIP         |
 | vkGetSwapchainImagesKHR                          | ⚙️ WIP         |
 | vkInvalidateMappedMemoryRanges                   | ⚙️ WIP         |
-| vkMapMemory                                      | ⚙️ WIP         |
+| vkMapMemory                                      | ✅ Implemented |
 | vkMergePipelineCaches                            | ⚙️ WIP         |
 | vkQueueBindSparse                                | ⚙️ WIP         |
 | vkQueuePresentKHR                                | ⚙️ WIP         |
 | vkQueueSubmit                                    | ✅ Implemented |
-| vkQueueWaitIdle                                  | ⚙️ WIP         |
+| vkQueueWaitIdle                                  | ✅ Implemented |
-| vkResetCommandBuffer                             | ⚙️ WIP         |
+| vkResetCommandBuffer                             | ✅ Implemented |
 | vkResetCommandPool                               | ✅ Implemented |
 | vkResetDescriptorPool                            | ⚙️ WIP         |
 | vkResetEvent                                     | ⚙️ WIP         |
-| vkResetFences                                    | ⚙️ WIP         |
+| vkResetFences                                    | ✅ Implemented |
 | vkResetQueryPool                                 | ⚙️ WIP         |
 | vkSetEvent                                       | ⚙️ WIP         |
-| vkUnmapMemory                                    | ⚙️ WIP         |
+| vkUnmapMemory                                    | ✅ Implemented |
 | vkUpdateDescriptorSets                           | ⚙️ WIP         |
-| vkWaitForFences                                  | ⚙️ WIP         |
+| vkWaitForFences                                  | ✅ Implemented |
 
 </details>
 
@@ -471,9 +473,9 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkBindImageMemory                                | ✅ Implemented |
 | vkCmdBeginQuery                                  | ⚙️ WIP         |
 | vkCmdBeginRenderPass                             | ⚙️ WIP         |
-| vkCmdBindDescriptorSets                          | ⚙️ WIP         |
+| vkCmdBindDescriptorSets                          | ✅ Implemented |
 | vkCmdBindIndexBuffer                             | ⚙️ WIP         |
-| vkCmdBindPipeline                                | ⚙️ WIP         |
+| vkCmdBindPipeline                                | ✅ Implemented |
 | vkCmdBindVertexBuffers                           | ⚙️ WIP         |
 | vkCmdBlitImage                                   | ⚙️ WIP         |
 | vkCmdClearAttachments                            | ⚙️ WIP         |
@@ -484,8 +486,8 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCmdCopyImage                                   | ✅ Implemented |
 | vkCmdCopyImageToBuffer                           | ✅ Implemented |
 | vkCmdCopyQueryPoolResults                        | ⚙️ WIP         |
-| vkCmdDispatch                                    | ⚙️ WIP         |
+| vkCmdDispatch                                    | ✅ Implemented |
-| vkCmdDispatchBaseKHR                             | ⚙️ WIP         |
+| vkCmdDispatchBaseKHR                             | ✅ Implemented |
 | vkCmdDispatchIndirect                            | ⚙️ WIP         |
 | vkCmdDraw                                        | ⚙️ WIP         |
 | vkCmdDrawIndexed                                 | ⚙️ WIP         |
@@ -518,9 +520,9 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCreateBuffer                                   | ✅ Implemented |
 | vkCreateBufferView                               | ⚙️ WIP         |
 | vkCreateCommandPool                              | ✅ Implemented |
-| vkCreateComputePipelines                         | ⚙️ WIP         |
+| vkCreateComputePipelines                         | ✅ Implemented |
-| vkCreateDescriptorPool                           | ⚙️ WIP         |
+| vkCreateDescriptorPool                           | ✅ Implemented |
-| vkCreateDescriptorSetLayout                      | ⚙️ WIP         |
+| vkCreateDescriptorSetLayout                      | ✅ Implemented |
 | vkCreateDevice                                   | ✅ Implemented |
 | vkCreateEvent                                    | ⚙️ WIP         |
 | vkCreateFence                                    | ✅ Implemented |
@@ -535,17 +537,17 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkCreateRenderPass                               | ⚙️ WIP         |
 | vkCreateSampler                                  | ⚙️ WIP         |
 | vkCreateSemaphore                                | ✅ Implemented |
-| vkCreateShaderModule                             | ⚙️ WIP         |
+| vkCreateShaderModule                             | ✅ Implemented |
-| vkCreateSwapchainKHR                             | ⚙️ WIP         |
+| vkCreateSwapchainKHR                             | ✅ Implemented |
-| vkCreateWaylandSurfaceKHR                        | ⚙️ WIP         |
+| vkCreateWaylandSurfaceKHR                        | ✅ Implemented |
 | vkCreateWin32SurfaceKHR                          | ⚙️ WIP         |
-| vkCreateXcbSurfaceKHR                            | ⚙️ WIP         |
+| vkCreateXcbSurfaceKHR                            | ✅ Implemented |
-| vkCreateXlibSurfaceKHR                           | ⚙️ WIP         |
+| vkCreateXlibSurfaceKHR                           | ✅ Implemented |
 | vkDestroyBuffer                                  | ✅ Implemented |
 | vkDestroyBufferView                              | ⚙️ WIP         |
 | vkDestroyCommandPool                             | ✅ Implemented |
-| vkDestroyDescriptorPool                          | ⚙️ WIP         |
+| vkDestroyDescriptorPool                          | ✅ Implemented |
-| vkDestroyDescriptorSetLayout                     | ⚙️ WIP         |
+| vkDestroyDescriptorSetLayout                     | ✅ Implemented |
 | vkDestroyDevice                                  | ✅ Implemented |
 | vkDestroyEvent                                   | ⚙️ WIP         |
 | vkDestroyFence                                   | ✅ Implemented |
@@ -553,16 +555,16 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkDestroyImage                                   | ✅ Implemented |
 | vkDestroyImageView                               | ⚙️ WIP         |
 | vkDestroyInstance                                | ✅ Implemented |
-| vkDestroyPipeline                                | ⚙️ WIP         |
+| vkDestroyPipeline                                | ✅ Implemented |
 | vkDestroyPipelineCache                           | ⚙️ WIP         |
 | vkDestroyPipelineLayout                          | ⚙️ WIP         |
 | vkDestroyQueryPool                               | ⚙️ WIP         |
 | vkDestroyRenderPass                              | ⚙️ WIP         |
 | vkDestroySampler                                 | ⚙️ WIP         |
 | vkDestroySemaphore                               | ✅ Implemented |
-| vkDestroyShaderModule                            | ⚙️ WIP         |
+| vkDestroyShaderModule                            | ✅ Implemented |
-| vkDestroySurfaceKHR                              | ⚙️ WIP         |
+| vkDestroySurfaceKHR                              | ✅ Implemented |
-| vkDestroySwapchainKHR                            | ⚙️ WIP         |
+| vkDestroySwapchainKHR                            | ✅ Implemented |
 | vkDeviceWaitIdle                                 | ✅ Implemented |
 | vkEndCommandBuffer                               | ✅ Implemented |
 | vkEnumerateDeviceExtensionProperties             | ✅ Implemented |
@@ -573,7 +575,7 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkEnumeratePhysicalDevices                       | ✅ Implemented |
 | vkFlushMappedMemoryRanges                        | ✅ Implemented |
 | vkFreeCommandBuffers                             | ✅ Implemented |
-| vkFreeDescriptorSets                             | ⚙️ WIP         |
+| vkFreeDescriptorSets                             | ✅ Implemented |
 | vkFreeMemory                                     | ✅ Implemented |
 | vkGetBufferDeviceAddress                         | ⚙️ WIP         |
 | vkGetBufferDeviceAddressEXT                      | ⚙️ WIP         |
@@ -590,7 +592,7 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkGetImageMemoryRequirements                     | ⚙️ WIP         |
 | vkGetImageSparseMemoryRequirements               | ⚙️ WIP         |
 | vkGetImageSubresourceLayout                      | ⚙️ WIP         |
-| vkGetInstanceProcAddr                            | ⚙️ WIP         |
+| vkGetInstanceProcAddr                            | ✅ Implemented |
 | vkGetPhysicalDeviceFeatures                      | ✅ Implemented |
 | vkGetPhysicalDeviceFormatProperties              | ⚙️ WIP         |
 | vkGetPhysicalDeviceImageFormatProperties         | ⚙️ WIP         |
@@ -619,7 +621,7 @@ Assume thou that functions lacking in this array are, for now, not intended to b
 | vkQueueWaitIdle                                  | ✅ Implemented |
 | vkResetCommandBuffer                             | ✅ Implemented |
 | vkResetCommandPool                               | ✅ Implemented |
-| vkResetDescriptorPool                            | ⚙️ WIP         |
+| vkResetDescriptorPool                            | ✅ Implemented |
 | vkResetEvent                                     | ⚙️ WIP         |
 | vkResetFences                                    | ✅ Implemented |
 | vkResetQueryPool                                 | ⚙️ WIP         |

build.zig

@@ -11,46 +11,43 @@ const ImplementationDesc = struct {
     vulkan_version: std.SemanticVersion,
     custom: ?*const fn (
         *std.Build,
+        *Step.Options,
         *Step.Compile,
         *std.Build.Module,
         *std.Build.Module,
         *std.Build.Module,
         *std.Build.Module,
+        *std.Build.Module,
         std.Build.ResolvedTarget,
         std.builtin.OptimizeMode,
         bool,
     ) anyerror!void = null,
-    options: ?*const fn (*std.Build, *Step.Options) anyerror!void = null,
 };
 
 const implementations = [_]ImplementationDesc{
-    .{
-        .name = "ape",
-        .icd_name = "ape",
-        .root_source_file = "src/ape/lib.zig",
-        .vulkan_version = .{ .major = 1, .minor = 0, .patch = 0 },
-        .custom = customApe,
-    },
     .{
         .name = "soft",
         .root_source_file = "src/software/lib.zig",
         .vulkan_version = .{ .major = 1, .minor = 0, .patch = 0 },
         .custom = customSoft,
-        .options = optionsSoft,
     },
     .{
         .name = "flint",
         .root_source_file = "src/intel/lib.zig",
         .vulkan_version = .{ .major = 1, .minor = 0, .patch = 0 },
         .custom = customFlint,
-        .options = optionsFlint,
     },
     .{
         .name = "phi",
         .root_source_file = "src/phi/lib.zig",
         .vulkan_version = .{ .major = 1, .minor = 0, .patch = 0 },
         .custom = customPhi,
-        .options = optionsPhi,
+    },
+    .{
+        .name = "ape",
+        .icd_name = "ape",
+        .root_source_file = "src/ape/lib.zig",
+        .vulkan_version = .{ .major = 1, .minor = 0, .patch = 0 },
     },
 };
 
@@ -129,6 +126,7 @@ pub fn build(b: *std.Build) !void {
     base_mod.addImport("vulkan", vulkan);
     base_mod.addImport("zmath", zmath);
     base_mod.addImport("drm", drm);
+    base_mod.addImport("shader_ir", ir_mod);
 
     const base_c_includes = b.addTranslateC(.{
         .root_source_file = b.path("src/vulkan/c_includes.h"),
@@ -149,7 +147,8 @@ pub fn build(b: *std.Build) !void {
 
     const use_llvm = b.option(bool, "use-llvm", "LLVM build") orelse (b.release_mode != .off);
 
-    for (implementations) |impl| {
+    var implementation_modules: [implementations.len]*std.Build.Module = undefined;
+    for (implementations, 0..) |impl, impl_index| {
         const lib_mod = b.createModule(.{
             .root_source_file = b.path(impl.root_source_file),
             .target = target,
@@ -161,6 +160,7 @@ pub fn build(b: *std.Build) !void {
             },
         });
 
+        implementation_modules[impl_index] = lib_mod;
         lib_mod.addSystemIncludePath(vulkan_headers.path("include"));
 
         const lib = b.addLibrary(.{
@@ -172,12 +172,11 @@ pub fn build(b: *std.Build) !void {
 
         options.addOption(std.SemanticVersion, b.fmt("{s}_vulkan_version", .{impl.name}), impl.vulkan_version);
 
-        if (impl.custom) |func| {
+        if (std.mem.eql(u8, impl.name, "ape")) {
-            func(b, lib, lib_mod, base_mod, vulkan, base_c_mod, target, optimize, use_llvm) catch continue;
+            for (implementations[0..impl_index], implementation_modules[0..impl_index]) |child_impl, child_mod|
-        }
+                lib_mod.addImport(child_impl.name, child_mod);
-
+        } else if (impl.custom) |func| {
-        if (impl.options) |func| {
+            func(b, options, lib, lib_mod, base_mod, vulkan, base_c_mod, ir_mod, target, optimize, use_llvm) catch continue;
-            func(b, options) catch continue;
         }
 
         const icd_file = b.addWriteFile(
@@ -219,8 +218,10 @@ pub fn build(b: *std.Build) !void {
         test_step.dependOn(&run_tests.step);
 
         inline for (std.enums.values(RunningMode)) |mode| {
-            (try addCTS(b, target, &impl, lib, mode)).dependOn(&lib_install.step);
+            if (addCTS(b, target, &impl, lib, mode) catch null) |step|
-            (try addMultithreadedCTS(b, target, &impl, lib, mode)).dependOn(&lib_install.step);
+                step.dependOn(&lib_install.step);
+            if (addMultithreadedCTS(b, target, &impl, lib, mode) catch null) |step|
+                step.dependOn(&lib_install.step);
         }
 
         const impl_autodoc_test = b.addObject(.{
@@ -256,12 +257,17 @@ pub fn build(b: *std.Build) !void {
 }
 
 fn addCTS(b: *std.Build, target: std.Build.ResolvedTarget, impl: *const ImplementationDesc, impl_lib: *Step.Compile, comptime mode: RunningMode) !*Step {
+    const arch = if (target.query.cpu_arch) |arch| arch else builtin.cpu.arch;
+    if (!arch.isX86())
+        return error.NoCTSForPlatform;
+
     const cts = b.dependency("cts_bin", .{});
 
     const cts_exe_name = cts.path(b.fmt("deqp-vk-{s}", .{
         switch (if (target.query.os_tag) |tag| tag else builtin.target.os.tag) {
             .linux => "linux.x86_64",
             .windows => "windows.exe",
+            .macos => "macos.x86_64",
             else => return error.NoCTSForPlatform,
         },
     }));
@@ -333,12 +339,17 @@ fn addCTS(b: *std.Build, target: std.Build.ResolvedTarget, impl: *const Implemen
 }
 
 fn addMultithreadedCTS(b: *std.Build, target: std.Build.ResolvedTarget, impl: *const ImplementationDesc, impl_lib: *Step.Compile, comptime mode: RunningMode) !*Step {
+    const arch = if (target.query.cpu_arch) |arch| arch else builtin.cpu.arch;
+    if (!arch.isX86())
+        return error.NoCTSForPlatform;
+
     const cts = b.dependency("cts_bin", .{});
 
     const cts_exe_name = cts.path(b.fmt("deqp-vk-{s}", .{
         switch (if (target.query.os_tag) |tag| tag else builtin.target.os.tag) {
             .linux => "linux.x86_64",
             .windows => "windows.exe",
+            .macos => "macos.x86_64",
             else => return error.NoCTSForPlatform,
         },
     }));
@@ -426,50 +437,17 @@ fn addMultithreadedCTS(b: *std.Build, target: std.Build.ResolvedTarget, impl: *c
     return &run.step;
 }
 
-// Ape specialized functions
-
-fn customApe(
-    b: *std.Build,
-    lib: *Step.Compile,
-    lib_mod: *std.Build.Module,
-    base_mod: *std.Build.Module,
-    vulkan: *std.Build.Module,
-    base_c_mod: *std.Build.Module,
-    target: std.Build.ResolvedTarget,
-    optimize: std.builtin.OptimizeMode,
-    use_llvm: bool,
-) !void {
-    for (implementations) |impl| {
-        if (std.mem.eql(u8, impl.name, "ape"))
-            continue;
-
-        const mod = b.createModule(.{
-            .root_source_file = b.path(impl.root_source_file),
-            .target = target,
-            .optimize = optimize,
-            .imports = &.{
-                .{ .name = "base", .module = base_mod },
-                .{ .name = "vulkan", .module = vulkan },
-            },
-        });
-
-        if (impl.custom) |func| {
-            func(b, lib, mod, base_mod, vulkan, base_c_mod, target, optimize, use_llvm) catch continue;
-        }
-
-        lib_mod.addImport(impl.name, mod);
-    }
-}
-
 // Soft specialized functions
 
 fn customSoft(
     b: *std.Build,
+    options: *Step.Options,
     _: *Step.Compile,
     lib_mod: *std.Build.Module,
     _: *std.Build.Module,
     _: *std.Build.Module,
     base_c_mod: *std.Build.Module,
+    shader_ir_mod: *std.Build.Module,
     target: std.Build.ResolvedTarget,
     optimize: std.builtin.OptimizeMode,
     use_llvm: bool,
@@ -482,16 +460,17 @@ fn customSoft(
 
     lib_mod.addImport("soft_c", base_c_mod);
     lib_mod.addImport("spv", spv.module("spv"));
-}
+    lib_mod.addImport("shader_ir", shader_ir_mod);
 
-fn optionsSoft(b: *std.Build, options: *Step.Options) !void {
     const single_threaded_option = b.option(bool, "soft-single-threaded", "Single threaded runtime mode") orelse false;
+    const ir_interpreter_option = b.option(bool, "soft-ir-interpreter", "Enable the experimental backend-agnostic IR interpreter") orelse false;
     const shaders_simd_option = b.option(bool, "soft-shader-simd", "Shaders SIMD acceleration") orelse true;
     const compute_dump_early_results_table_option = b.option(u32, "soft-compute-dump-early-results-table", "Dump compute shaders results table before invocation");
     const compute_dump_final_results_table_option = b.option(u32, "soft-compute-dump-final-results-table", "Dump compute shaders results table after invocation");
     const approxiamte_rgb_option = b.option(bool, "soft-approximates-rgb", "Approximate sRGB <-> RGB conversions") orelse true;
 
     options.addOption(bool, "soft_single_threaded", single_threaded_option);
+    options.addOption(bool, "soft_ir_interpreter", ir_interpreter_option);
     options.addOption(bool, "soft_shaders_simd", shaders_simd_option);
     options.addOption(?u32, "soft_compute_dump_early_results_table", compute_dump_early_results_table_option);
     options.addOption(?u32, "soft_compute_dump_final_results_table", compute_dump_final_results_table_option);
@@ -502,42 +481,59 @@ fn optionsSoft(b: *std.Build, options: *Step.Options) !void {
 
 fn customFlint(
     b: *std.Build,
+    options: *Step.Options,
     _: *Step.Compile,
     lib_mod: *std.Build.Module,
     _: *std.Build.Module,
     _: *std.Build.Module,
     base_c_mod: *std.Build.Module,
-    target: std.Build.ResolvedTarget,
+    shader_ir_mod: *std.Build.Module,
-    optimize: std.builtin.OptimizeMode,
+    _: std.Build.ResolvedTarget,
+    _: std.builtin.OptimizeMode,
     _: bool,
 ) !void {
     lib_mod.addImport("intel_c", base_c_mod);
-    lib_mod.addImport("shader_ir", b.createModule(.{
+    lib_mod.addImport("shader_ir", shader_ir_mod);
-        .root_source_file = b.path("src/compiler/root.zig"),
-        .target = target,
-        .optimize = optimize,
-    }));
-}
 
-fn optionsFlint(b: *std.Build, options: *Step.Options) !void {
+    const dump_common_ir = b.option(bool, "flint-dump-common-ir", "Print backend-agnostic shader IR after translation") orelse false;
-    _ = b;
+    const dump_ir = b.option(bool, "flint-dump-ir", "Print final Flint IR after backend lowering") orelse false;
-    _ = options;
+
+    options.addOption(bool, "flint_dump_common_ir", dump_common_ir);
+    options.addOption(bool, "flint_dump_ir", dump_ir);
 }
 
 // Phi specialized functions
 
 fn customPhi(
     b: *std.Build,
+    options: *Step.Options,
     lib: *Step.Compile,
     lib_mod: *std.Build.Module,
     _: *std.Build.Module,
     _: *std.Build.Module,
     base_c_mod: *std.Build.Module,
+    shader_ir_mod: *std.Build.Module,
     target: std.Build.ResolvedTarget,
     optimize: std.builtin.OptimizeMode,
     use_llvm: bool,
 ) !void {
+    const daemon_remote_path = b.option(
+        []const u8,
+        "phi-daemon-remote-path",
+        "Path where the Xeon Phi daemon is copied on the card",
+    ) orelse "/tmp/phi_device.mic";
+
+    const daemon_host_prefix = b.option(
+        []const u8,
+        "phi-daemon-host-prefix",
+        "Host prefix used to reach cards over ssh/scp; card N uses <prefix>N",
+    ) orelse "mic";
+
+    options.addOption([]const u8, "phi_daemon_remote_path", daemon_remote_path);
+    options.addOption([]const u8, "phi_daemon_host_prefix", daemon_host_prefix);
+
     lib_mod.addImport("phi_c", base_c_mod);
+    lib_mod.addImport("shader_ir", shader_ir_mod);
 
     const miclib = b.lazyDependency("miclib", .{
         .target = target,
@@ -556,20 +552,10 @@ fn customPhi(
 
     lib_mod.addImport("phi_protocol_c", phi_protocol_c.createModule());
 
-    // To avoid duplicated options due to Ape's custom function
-    if (!std.mem.eql(u8, lib.name, "vulkan_phi")) {
-        const daemon = try addPhiCardDaemon(b, optimize, "k1om-mpss-linux-gcc", null);
-        const embedded_daemon = addEmbeddedPhiDaemon(b, daemon);
-        lib_mod.addAnonymousImport("phi_daemon", .{
-            .root_source_file = embedded_daemon,
-        });
-        return;
-    }
-
     const build_card = b.option(
         bool,
         "phi-build-daemon",
-        "Build Xeon Phi card daemon",
+        "Build the Phi device daemon",
     ) orelse true;
 
     if (!build_card)
@@ -587,7 +573,7 @@ fn customPhi(
         "MPSS sysroot path",
     );
 
-    const daemon = try addPhiCardDaemon(b, optimize, cc, sysroot);
+    const daemon = try addPhiDaemon(b, optimize, cc, sysroot);
     const install_daemon = b.addInstallFile(daemon, "lib/phi_device.mic");
     lib.step.dependOn(&install_daemon.step);
 
@@ -597,36 +583,19 @@ fn customPhi(
     });
 }
 
-fn optionsPhi(b: *std.Build, options: *Step.Options) !void {
+fn addPhiDaemonCompilerArgs(
-    const daemon_remote_path = b.option(
+    cmd: *Step.Run,
-        []const u8,
-        "phi-daemon-remote-path",
-        "Path where the Xeon Phi daemon is copied on the card",
-    ) orelse "/tmp/phi_device.mic";
-
-    const daemon_host_prefix = b.option(
-        []const u8,
-        "phi-daemon-host-prefix",
-        "Host prefix used to reach cards over ssh/scp; card N uses <prefix>N",
-    ) orelse "mic";
-
-    options.addOption([]const u8, "phi_daemon_remote_path", daemon_remote_path);
-    options.addOption([]const u8, "phi_daemon_host_prefix", daemon_host_prefix);
-}
-
-fn addPhiCardDaemon(
     b: *std.Build,
     optimize: std.builtin.OptimizeMode,
-    cc: []const u8,
     sysroot: ?[]const u8,
-) !std.Build.LazyPath {
+) void {
-    const cmd = b.addSystemCommand(&.{cc});
-
     cmd.addArgs(&.{
         "-std=c11",
         "-Wall",
         "-Wextra",
+        "-Werror",
         "-Wno-unused-parameter",
+        "-Wno-unused-variable",
         "-pthread",
     });
 
@@ -643,29 +612,61 @@ fn addPhiCardDaemon(
     switch (optimize) {
         .Debug => cmd.addArgs(&.{ "-O0", "-g3" }),
         .ReleaseSafe => cmd.addArgs(&.{ "-O2", "-g", "-DNDEBUG" }),
-        .ReleaseFast => cmd.addArgs(&.{ "-O3", "-DNDEBUG" }),
+        .ReleaseFast => cmd.addArgs(&.{ "-O3", "-DNDEBUG", "-DNOLOGS" }),
         .ReleaseSmall => cmd.addArgs(&.{ "-Os", "-DNDEBUG" }),
     }
+}
+
+fn addPhiDaemon(b: *std.Build, optimize: std.builtin.OptimizeMode, cc: []const u8, sysroot: ?[]const u8) !std.Build.LazyPath {
+    const cmd = b.addSystemCommand(&.{cc});
+    addPhiDaemonCompilerArgs(cmd, b, optimize, sysroot);
 
     const sources = [_][]const u8{
         "src/phi/mic/main.c",
+        "src/phi/mic/Blitter.c",
+        "src/phi/mic/BlitFormats.c",
         "src/phi/mic/Buffer.c",
         "src/phi/mic/CommandBuffer.c",
         "src/phi/mic/Daemon.c",
+        "src/phi/mic/Image.c",
         "src/phi/mic/Logger.c",
         "src/phi/mic/Memory.c",
-        // Add new files here
+        "src/phi/mic/Queue.c",
+        "src/phi/mic/Transport.c",
+        "src/phi/mic/WorkerPool.c",
+        // Add non-AVX files here
     };
 
     for (sources) |source| {
         cmd.addFileArg(b.path(source));
     }
 
-    cmd.addArgs(&.{
+    // Keep KNC AVX-512/IMCI code in separate translation units. The GCC port
-        "-lscif",
+    // in use must not compile the daemon's scalar/control code with -mavx512f
-        "-o",
+    const avx_sources = [_][]const u8{
-    });
+        "src/phi/mic/avx/Blit.c",
+        "src/phi/mic/avx/Copy.c",
+        "src/phi/mic/avx/Fill.c",
+        // Add AVX files here
+    };
 
+    for (avx_sources, 0..) |source, index| {
+        const avx_cmd = b.addSystemCommand(&.{cc});
+        addPhiDaemonCompilerArgs(avx_cmd, b, optimize, sysroot);
+
+        avx_cmd.addArg("-mavx512f");
+        avx_cmd.addArg("-c");
+        avx_cmd.addFileArg(b.path(source));
+        avx_cmd.addArg("-o");
+
+        const avx_object = avx_cmd.addOutputFileArg(
+            b.fmt("phi_avx_{d}.o", .{index}),
+        );
+
+        cmd.addFileArg(avx_object);
+    }
+
+    cmd.addArgs(&.{ "-lscif", "-lm", "-o" });
     return cmd.addOutputFileArg("phi_device.mic");
 }
 

build.zig.zon

@@ -23,8 +23,8 @@
             .hash = "zmath-0.11.0-dev-wjwivdMsAwD-xaLj76YHUq3t9JDH-X16xuMTmnDzqbu2",
         },
         .cts_bin = .{
-            .url = "git+https://git.kbz8.me/kbz_8/Vulkan-CTS-bin.git#b316a134bc0aa7ac21d9c57a1df588809824dcdc",
+            .url = "git+https://git.kbz8.me/kbz_8/Vulkan-CTS-bin.git#190a251847dd8d41b1cd05493d5e2b5fd3d4cd32",
-            .hash = "N-V-__8AAF9uOh0I4P_99za7N822J3JwsDaqONrFVrcEQo59",
+            .hash = "N-V-__8AAF_AmSKsM1ucsQiHOIC71T6pFAneEsOu2hzlc2IA",
         },
         .drm = .{
             .url = "git+https://github.com/Kbz-8/zig-drm#409f58daa8f5174b2fcb8897f1c30f0b0729b611",

compile_flags.txt

@@ -1,5 +1,6 @@
 -xc
 -std=c11
+-mavx512f
 -Isrc/phi/shared
 -Isrc/phi/mic
 -isystem/opt/mpss/3.8.6/sysroots/k1om-mpss-linux/usr/include/

mirror-test.txt

@@ -1 +0,0 @@
-tests

src/compiler/README.md

@@ -30,7 +30,7 @@ The printer uses these prefixes:
 | Prefix  | Meaning                                                          | Example               |
 | ------- | ---------------------------------------------------------------- | --------------------- |
 | `%id`   | An SSA value, whether constant, parameter, or instruction result | `%3`, `%merged_value` |
-| `@name` | A function or interface declaration                              | `@main`, `@out_color` |
+| `@name` | A function, interface, or resource declaration                   | `@main`, `@out_color` |
 | `.name` | A basic block                                                    | `.entry`, `.merge`    |
 | `#N`    | A constant-store identity used within composite constants        | `#2`                  |
 
@@ -55,6 +55,7 @@ The outer structure has this shape:
 shader <stage> @<entry-point>
 {
     <interface declarations>
+    <resource declarations>
     <constant declarations>
 
     fn @<name>(<parameters>) -> <type>
@@ -67,8 +68,8 @@ shader <stage> @<entry-point>
 }
 ```
 
-Execution modes, resources, source locations, and structured-control metadata
+Execution modes, source locations, and structured-control metadata exist in
-exist in memory, but the printer does not display them yet.
+memory, but the printer does not display them yet.
 
 ## Parsing
 
@@ -113,6 +114,25 @@ The current resource kinds are `uniform_buffer`, `storage_buffer`,
 `sampled_image`, `storage_image`, and `sampler`. A resource handle may also
 carry an optional data type in memory, although the printer omits that type.
 
+## Resources
+
+Resources are declared at module scope. Every `ResourceKind` uses the same
+`set` and `binding` syntax:
+
+```text
+@name: TYPE = storage_buffer[set(N), binding(N)]
+```
+
+The declaration `TYPE` is the storage buffer's block or payload aggregate type;
+it does not constrain the type of each byte-addressed access. Both operations
+accept storage buffers, and byte offsets must have a scalar unsigned integer
+type. Access values may be integer or floating-point scalars or vectors thereof.
+
+```text
+%value: TYPE = load_buffer @name, %offset
+store_buffer @name, %offset, %value
+```
+
 ## Constants
 
 Constants live at module scope and also have ordinary numeric or named `%id` value identities.
@@ -188,8 +208,8 @@ metadata. They are not terminators and do not create graph edges themselves.
 ## Common instruction rules
 
 An instruction belongs to one block, has zero or one result, and may carry a
-source location. Except for `store_interface` and `call`, current operations are
+source location. Except for `store_interface`, `store_buffer`, and `call`, current
-treated as side-effect free by the rewriter. A block's terminator is stored
+operations are treated as side-effect free by the rewriter. A block's terminator is stored
 separately from its ordinary instructions.
 
 Most arithmetic operations are intended for scalars or vectors of their named
@@ -379,6 +399,28 @@ The stored value must equal the interface variable's type. As with
 `load_interface`, an optional unprinted `element_index` is reserved for later
 arrayed-interface work. This operation has side effects.
 
+### `load_buffer`
+
+Reads a numeric scalar or vector at an explicit byte offset. The resource must
+be a storage buffer, the byte offset must be a scalar unsigned integer, and the
+instruction must have a result. The result type is independent of the resource's
+block or payload aggregate type.
+
+```text
+%value: u32 = load_buffer @data, %offset
+```
+
+### `store_buffer`
+
+Writes a numeric scalar or vector to a storage buffer at an explicit byte
+offset. It produces no SSA result, and both the unsigned integer offset and
+stored value are ordinary value uses. The value type is independent of the
+resource's block or payload aggregate type. This operation has side effects.
+
+```text
+store_buffer @data, %offset, %value
+```
+
 ### `call`
 
 Invokes another IR function. Arguments must match the callee's parameters in
@@ -573,14 +615,39 @@ final large-shader implementation.
 
 ## SPIR-V frontend
 
-The compiler currently provides a word parser and an initial translator in
+The compiler currently provides a word parser, an owned `SourceModule`, and an
-`spirv/`. The parser validates the header, word counts, truncation, and literal
+initial translator in `spirv/`. The parser validates the header, word counts,
-strings. The translator selects one entry point and lowers a defined subset:
+truncation, and literal strings. `SourceModule` copies and retains validated
+SPIR-V so API objects can instantiate multiple entry points without borrowing
+application memory.
+
+Use `translator.instantiate` when retaining a source module:
+
+```zig
+var source = try ir.spirv.SourceModule.init(allocator, words);
+defer source.deinit(allocator);
+
+var module = try ir.spirv.translator.instantiate(allocator, &source, .{
+    .entry_point = "main",
+    .stage = .compute,
+    .specializations = &.{.{
+        .constant_id = 7, // SPIR-V SpecId
+        .data = std.mem.asBytes(&workgroup_width),
+    }},
+});
+defer module.deinit();
+```
+
+`translator.translate` remains a convenience wrapper for borrowed words. Each
+translation selects one entry point and returns an independent mutable IR
+module. The translator lowers a defined subset:
 
 - Vertex, fragment, and compute stages.
 - Basic scalar, vector, array, structure, pointer, and function types.
-- Ordinary and composite constants; unapplied specialization constants are
+- Ordinary constants plus scalar boolean, integer, and floating-point
-  refused.
+  specialization constants selected through `SpecId`. Missing overrides use the
+  SPIR-V defaults, and specialization composites are rebuilt from their
+  specialized elements. `OpSpecConstantOp` is not evaluated yet.
 - Functions, blocks, branches, structured merge marks, and returns.
 - `OpPhi` into block parameters and edge arguments.
 - The arithmetic, comparison, select, bitcast, and composite operations named

src/compiler/ir/Builder.zig

@@ -169,6 +169,16 @@ pub fn addInterfaceVariable(
     });
 }
 
+pub fn addResource(self: *Self, ty: ids.TypeId, kind: type_ir.ResourceKind, set: u32, binding: u32, name: ?[]const u8) !ids.ResourceId {
+    return self.module.resources.add(self.module.allocator(), .{
+        .kind = kind,
+        .set = set,
+        .binding = binding,
+        .type = ty,
+        .name = try self.copyName(name),
+    });
+}
+
 pub fn edge(self: *Self, target: ids.BlockId, arguments: []const ids.ValueId) !module_ir.Edge {
     return .{
         .target = target,
@@ -225,3 +235,116 @@ fn constantEql(a: constant_ir.ConstantValue, b: constant_ir.ConstantValue) bool
         .composite => |value| b == .composite and std.mem.eql(ids.ConstantId, value, b.composite),
     };
 }
+
+test "Builder: generation" {
+    const cfg = @import("cfg.zig");
+    const parser = @import("parser/parser.zig");
+    const printer = @import("printer.zig");
+    const validator = @import("validator/validator.zig");
+
+    // shader vertex @main
+    // {
+    //     @color: vec4[f32] = input[location(0), component(0), index(0)]
+    //     @out_color: vec4[f32] = output[location(0), component(0), index(0)]
+    //     %0: constant bool = true
+    //     %1: constant f32 = bits(0x3f800000)
+    //
+    //     fn @main() -> void
+    //     {
+    //         .entry():
+    //             %3: vec4[f32] = load_interface @color
+    //             conditional_branch %0, .pass(), .merge(%3)
+    //
+    //         .pass():
+    //             %4: vec4[f32] = composite_construct %1, %1, %1, %1
+    //             branch .merge(%4)
+    //
+    //         .merge(%2: vec4[f32]):
+    //             store_interface @out_color, %2
+    //             return
+    //     }
+    // }
+
+    var module = module_ir.Module.init(std.testing.allocator, .vertex);
+    defer module.deinit();
+    var builder = Self.init(&module);
+
+    const void_type = try builder.internType(.void);
+    const bool_type = try builder.internType(.boolean);
+    const f32_type = try builder.internType(.{ .floating = .{ .bits = 32 } });
+    const duplicate_f32 = try builder.internType(.{ .floating = .{ .bits = 32 } });
+    try std.testing.expectEqual(f32_type, duplicate_f32);
+    const vec4_type = try builder.internType(.{ .vector = .{ .element_type = f32_type, .length = 4 } });
+
+    const true_value = try builder.internConstant(bool_type, .{ .boolean = true });
+    const one = try builder.internConstant(f32_type, .{ .float_bits = @as(u32, @bitCast(@as(f32, 1.0))) });
+
+    const input = try builder.addInterfaceVariable(vec4_type, .input, .{ .location = .{ .location = 0 } }, "color");
+    const output = try builder.addInterfaceVariable(vec4_type, .output, .{ .location = .{ .location = 0 } }, "out_color");
+    const main = try builder.addFunction(void_type, "main");
+    builder.setEntryPoint(main);
+    const entry = try builder.addBlock(main, "entry");
+    const pass = try builder.addBlock(main, "pass");
+    const merge = try builder.addBlock(main, "merge");
+    const merged = try builder.addBlockParameter(merge, vec4_type, "merged");
+
+    const loaded = (try builder.appendInstruction(entry, vec4_type, .{
+        .load_interface = .{ .variable = input },
+    }, "loaded")).?;
+    try builder.setTerminator(entry, .{ .conditional_branch = .{
+        .condition = true_value,
+        .true_edge = try builder.edge(pass, &.{}),
+        .false_edge = try builder.edge(merge, &.{loaded}),
+    } });
+
+    const splat = (try builder.appendInstruction(pass, vec4_type, .{
+        .composite_construct = .{ .elements = &.{ one, one, one, one } },
+    }, "white")).?;
+    try builder.setTerminator(pass, .{ .branch = try builder.edge(merge, &.{splat}) });
+    _ = try builder.appendInstruction(merge, null, .{
+        .store_interface = .{ .variable = output, .value = merged },
+    }, null);
+    try builder.setTerminator(merge, .return_void);
+
+    try validator.validate(&module);
+
+    var control_flow = try cfg.init(std.testing.allocator, &module, main);
+    defer control_flow.deinit();
+    try std.testing.expectEqual(@as(usize, 2), control_flow.predecessors(merge).?.len);
+    try std.testing.expect(control_flow.dominates(entry, merge));
+    try std.testing.expect(!control_flow.dominates(pass, merge));
+
+    const text = try printer.allocPrint(std.testing.allocator, &module);
+
+    defer std.testing.allocator.free(text);
+    try std.testing.expect(std.mem.indexOf(u8, text, "shader vertex @main") != null);
+    try std.testing.expect(std.mem.indexOf(u8, text, "@color: vec4[f32] = input[location(0), component(0), index(0)]") != null);
+    try std.testing.expect(std.mem.indexOf(u8, text, "@out_color: vec4[f32] = output[location(0), component(0), index(0)]") != null);
+    try std.testing.expect(std.mem.indexOf(u8, text, "conditional_branch %0, .pass(), .merge(%loaded)") != null);
+    try std.testing.expect(std.mem.indexOf(u8, text, ".merge(%merged: vec4[f32])") != null);
+    try std.testing.expect(std.mem.indexOf(u8, text, "store_interface @out_color, %merged") != null);
+
+    var parsed = try parser.parseString(std.testing.allocator, text);
+    defer parsed.deinit();
+    const round_trip = try printer.allocPrint(std.testing.allocator, &parsed);
+    defer std.testing.allocator.free(round_trip);
+    try std.testing.expectEqualStrings(text, round_trip);
+
+    const io = std.Options.debug_io;
+    const path = ".zig-cache/ir-parser-round-trip.ir";
+    const file = try std.Io.Dir.cwd().createFile(io, path, .{ .truncate = true });
+    {
+        defer file.close(io);
+        var file_buffer: [4096]u8 = @splat(0);
+        var file_writer = file.writer(io, &file_buffer);
+        try file_writer.interface.writeAll(text);
+        try file_writer.interface.flush();
+    }
+    defer std.Io.Dir.cwd().deleteFile(io, path) catch @panic("Caught an error while handling an error");
+
+    var parsed_file = try parser.parseFile(std.testing.allocator, io, path);
+    defer parsed_file.deinit();
+    const file_round_trip = try printer.allocPrint(std.testing.allocator, &parsed_file);
+    defer std.testing.allocator.free(file_round_trip);
+    try std.testing.expectEqualStrings(text, file_round_trip);
+}

src/compiler/ir/Rewriter.zig

@@ -2,6 +2,7 @@ const std = @import("std");
 const ids = @import("id.zig");
 const module_ir = @import("module.zig");
 const Builder = @import("Builder.zig");
+const validator = @import("validator/validator.zig");
 
 const Self = @This();
 
@@ -52,11 +53,11 @@ pub fn countUses(self: *const Self, value: ids.ValueId) usize {
 }
 
 pub fn replaceAllUses(self: *Self, old: ids.ValueId, replacement: ids.ValueId) Error!usize {
-    const old_value = self.module.values.get(old) orelse return error.InvalidValue;
+    const old_value = self.module.values.get(old) orelse return Error.InvalidValue;
-    const replacement_value = self.module.values.get(replacement) orelse return error.InvalidValue;
+    const replacement_value = self.module.values.get(replacement) orelse return Error.InvalidValue;
 
     if (old_value.type != replacement_value.type)
-        return error.TypeMismatch;
+        return Error.TypeMismatch;
 
     if (old == replacement)
         return 0;
@@ -77,17 +78,17 @@ pub fn replaceAllUses(self: *Self, old: ids.ValueId, replacement: ids.ValueId) E
 }
 
 pub fn eraseInstruction(self: *Self, instruction_id: ids.InstructionId) Error!void {
-    const instruction = self.module.instructions.get(instruction_id) orelse return error.InvalidInstruction;
+    const instruction = self.module.instructions.get(instruction_id) orelse return Error.InvalidInstruction;
 
     if (instruction.operation.hasSideEffects())
-        return error.SideEffectingInstruction;
+        return Error.SideEffectingInstruction;
 
     if (instruction.result) |result| {
         if (self.countUses(result) != 0)
-            return error.ResultStillUsed;
+            return Error.ResultStillUsed;
     }
 
-    const block = self.module.blocks.getMut(instruction.parent_block) orelse return error.InvalidBlock;
+    const block = self.module.blocks.getMut(instruction.parent_block) orelse return Error.InvalidBlock;
     var owned_index: ?usize = null;
 
     for (block.instructions.items, 0..) |candidate, index| {
@@ -97,29 +98,23 @@ pub fn eraseInstruction(self: *Self, instruction_id: ids.InstructionId) Error!vo
         }
     }
 
-    _ = block.instructions.orderedRemove(owned_index orelse return error.InstructionNotOwnedByBlock);
+    _ = block.instructions.orderedRemove(owned_index orelse return Error.InstructionNotOwnedByBlock);
     if (instruction.result) |result|
         _ = self.module.values.remove(result);
     _ = self.module.instructions.remove(instruction_id);
 }
 
-pub fn redirectEdges(
+pub fn redirectEdges(self: *Self, source: ids.BlockId, old_target: ids.BlockId, new_target: ids.BlockId, new_arguments: []const ids.ValueId) Error!usize {
-    self: *Self,
+    const source_block = self.module.blocks.get(source) orelse return Error.InvalidBlock;
-    source: ids.BlockId,
+    const target_block = self.module.blocks.get(new_target) orelse return Error.InvalidBlock;
-    old_target: ids.BlockId,
-    new_target: ids.BlockId,
-    new_arguments: []const ids.ValueId,
-) Error!usize {
-    const source_block = self.module.blocks.get(source) orelse return error.InvalidBlock;
-    const target_block = self.module.blocks.get(new_target) orelse return error.InvalidBlock;
 
     if (source_block.parent_function != target_block.parent_function)
-        return error.InvalidFunction;
+        return Error.InvalidFunction;
 
     try self.validateArguments(target_block, new_arguments);
 
     const mutable_source = self.module.blocks.getMut(source).?;
-    const terminator = if (mutable_source.terminator) |*value| value else return error.InvalidBlock;
+    const terminator = if (mutable_source.terminator) |*value| value else return Error.InvalidBlock;
 
     var count: usize = 0;
 
@@ -141,29 +136,23 @@ pub fn redirectEdges(
     return count;
 }
 
-pub fn addBlockParameter(
+pub fn addBlockParameter(self: *Self, block_id: ids.BlockId, ty: ids.TypeId, name: ?[]const u8, incoming: []const IncomingValue) Error!ids.ValueId {
-    self: *Self,
+    const block = self.module.blocks.get(block_id) orelse return Error.InvalidBlock;
-    block_id: ids.BlockId,
+    const function = self.module.functions.get(block.parent_function) orelse return Error.InvalidFunction;
-    ty: ids.TypeId,
-    name: ?[]const u8,
-    incoming: []const IncomingValue,
-) Error!ids.ValueId {
-    const block = self.module.blocks.get(block_id) orelse return error.InvalidBlock;
-    const function = self.module.functions.get(block.parent_function) orelse return error.InvalidFunction;
 
     for (incoming) |item| {
-        const value = self.module.values.get(item.value) orelse return error.InvalidValue;
+        const value = self.module.values.get(item.value) orelse return Error.InvalidValue;
 
         if (value.type != ty)
-            return error.TypeMismatch;
+            return Error.TypeMismatch;
 
         if (!functionHasEdgeTo(self.module, function, item.predecessor, block_id))
-            return error.UnexpectedIncomingValue;
+            return Error.UnexpectedIncomingValue;
     }
     for (function.blocks.items) |predecessor| {
         const edge_count = countEdgesTo(self.module.blocks.get(predecessor).?, block_id);
         if (edge_count != 0 and findIncoming(incoming, predecessor) == null)
-            return error.MissingIncomingValue;
+            return Error.MissingIncomingValue;
     }
 
     var builder = Builder.init(self.module);
@@ -177,19 +166,14 @@ pub fn addBlockParameter(
     return parameter;
 }
 
-pub fn removeBlockParameter(
+pub fn removeBlockParameter(self: *Self, block_id: ids.BlockId, parameter_index: usize, replacement: ids.ValueId) Error!void {
-    self: *Self,
+    const block = self.module.blocks.get(block_id) orelse return Error.InvalidBlock;
-    block_id: ids.BlockId,
+    if (parameter_index >= block.parameters.items.len) return Error.InvalidParameterIndex;
-    parameter_index: usize,
-    replacement: ids.ValueId,
-) Error!void {
-    const block = self.module.blocks.get(block_id) orelse return error.InvalidBlock;
-    if (parameter_index >= block.parameters.items.len) return error.InvalidParameterIndex;
     const parameter = block.parameters.items[parameter_index];
-    if (parameter == replacement) return error.InvalidValue;
+    if (parameter == replacement) return Error.InvalidValue;
     _ = try self.replaceAllUses(parameter, replacement);
 
-    const function = self.module.functions.get(block.parent_function) orelse return error.InvalidFunction;
+    const function = self.module.functions.get(block.parent_function) orelse return Error.InvalidFunction;
     for (function.blocks.items) |predecessor| {
         try self.removeArgumentFromEdges(predecessor, block_id, parameter_index);
     }
@@ -198,7 +182,7 @@ pub fn removeBlockParameter(
     _ = mutable_block.parameters.orderedRemove(parameter_index);
 
     for (mutable_block.parameters.items[parameter_index..], parameter_index..) |value_id, index| {
-        const value = self.module.values.getMut(value_id) orelse return error.InvalidValue;
+        const value = self.module.values.getMut(value_id) orelse return Error.InvalidValue;
         value.definition.block_parameter.index = @intCast(index);
     }
 
@@ -206,21 +190,15 @@ pub fn removeBlockParameter(
 }
 
 fn validateArguments(self: *const Self, target: *const module_ir.Block, arguments: []const ids.ValueId) Error!void {
-    if (arguments.len != target.parameters.items.len) return error.TypeMismatch;
+    if (arguments.len != target.parameters.items.len) return Error.TypeMismatch;
     for (arguments, target.parameters.items) |argument, parameter| {
-        const argument_value = self.module.values.get(argument) orelse return error.InvalidValue;
+        const argument_value = self.module.values.get(argument) orelse return Error.InvalidValue;
-        const parameter_value = self.module.values.get(parameter) orelse return error.InvalidValue;
+        const parameter_value = self.module.values.get(parameter) orelse return Error.InvalidValue;
-        if (argument_value.type != parameter_value.type) return error.TypeMismatch;
+        if (argument_value.type != parameter_value.type) return Error.TypeMismatch;
     }
 }
 
-fn redirectOne(
+fn redirectOne(self: *Self, edge: *module_ir.Edge, old_target: ids.BlockId, new_target: ids.BlockId, arguments: []const ids.ValueId) !bool {
-    self: *Self,
-    edge: *module_ir.Edge,
-    old_target: ids.BlockId,
-    new_target: ids.BlockId,
-    arguments: []const ids.ValueId,
-) !bool {
     if (edge.target != old_target)
         return false;
 
@@ -230,8 +208,8 @@ fn redirectOne(
 }
 
 fn appendArgumentToEdges(self: *Self, predecessor: ids.BlockId, target: ids.BlockId, value: ids.ValueId) !void {
-    const block = self.module.blocks.getMut(predecessor) orelse return error.InvalidBlock;
+    const block = self.module.blocks.getMut(predecessor) orelse return Error.InvalidBlock;
-    const terminator = if (block.terminator) |*item| item else return error.InvalidBlock;
+    const terminator = if (block.terminator) |*item| item else return Error.InvalidBlock;
 
     switch (terminator.*) {
         .branch => |*edge| {
@@ -257,8 +235,8 @@ fn appendEdgeArgument(self: *Self, edge: *module_ir.Edge, value: ids.ValueId) !v
 }
 
 fn removeArgumentFromEdges(self: *Self, predecessor: ids.BlockId, target: ids.BlockId, index: usize) !void {
-    const block = self.module.blocks.getMut(predecessor) orelse return error.InvalidBlock;
+    const block = self.module.blocks.getMut(predecessor) orelse return Error.InvalidBlock;
-    const terminator = if (block.terminator) |*item| item else return error.InvalidBlock;
+    const terminator = if (block.terminator) |*item| item else return Error.InvalidBlock;
 
     switch (terminator.*) {
         .branch => |*edge| {
@@ -278,7 +256,7 @@ fn removeArgumentFromEdges(self: *Self, predecessor: ids.BlockId, target: ids.Bl
 
 fn removeEdgeArgument(self: *Self, edge: *module_ir.Edge, index: usize) !void {
     if (index >= edge.arguments.len)
-        return error.InvalidParameterIndex;
+        return Error.InvalidParameterIndex;
 
     const arguments = try self.module.allocator().alloc(ids.ValueId, edge.arguments.len - 1);
     @memcpy(arguments[0..index], edge.arguments[0..index]);
@@ -299,12 +277,7 @@ fn findIncoming(incoming: []const IncomingValue, predecessor: ids.BlockId) ?ids.
     return null;
 }
 
-fn functionHasEdgeTo(
+fn functionHasEdgeTo(module: *const module_ir.Module, function: *const module_ir.Function, predecessor: ids.BlockId, target: ids.BlockId) bool {
-    module: *const module_ir.Module,
-    function: *const module_ir.Function,
-    predecessor: ids.BlockId,
-    target: ids.BlockId,
-) bool {
     for (function.blocks.items) |block_id| {
         if (block_id != predecessor)
             continue;
@@ -321,3 +294,135 @@ fn countEdgesTo(block: *const module_ir.Block, target: ids.BlockId) usize {
         else => 0,
     };
 }
+
+test "Rewriter: replace all ID uses, safely erase dead instruction" {
+    // shader compute @main
+    // {
+    //     %0: constant u32 = bits(0x1)
+    //     %1: constant u32 = bits(0x2)
+    //
+    //     fn @main() -> void
+    //     {
+    //         .entry():
+    //             %2: u32 = integer_add %0, %1
+    //             %3: u32 = integer_multiply %2, %1
+    //             return
+    //     }
+    // }
+
+    var module = module_ir.Module.init(std.testing.allocator, .compute);
+    defer module.deinit();
+
+    var builder = Builder.init(&module);
+
+    const void_type = try builder.internType(.void);
+    const u32_type = try builder.internType(.{ .integer = .{ .bits = 32, .signedness = .unsigned } });
+
+    const one = try builder.internConstant(u32_type, .{ .integer_bits = 1 });
+    const two = try builder.internConstant(u32_type, .{ .integer_bits = 2 });
+
+    const main = try builder.addFunction(void_type, "main");
+    builder.setEntryPoint(main);
+
+    const entry = try builder.addBlock(main, "entry");
+    const sum = (try builder.appendInstruction(entry, u32_type, .{
+        .binary = .{
+            .opcode = .integer_add,
+            .lhs = one,
+            .rhs = two,
+        },
+    }, null)).?;
+    _ = try builder.appendInstruction(entry, u32_type, .{
+        .binary = .{
+            .opcode = .integer_multiply,
+            .lhs = sum,
+            .rhs = two,
+        },
+    }, null);
+    try builder.setTerminator(entry, .return_void);
+
+    try validator.validate(&module);
+
+    const sum_instruction = module.values.get(sum).?.definition.instruction;
+    var rewriter = Self.init(&module);
+
+    try std.testing.expectEqual(@as(usize, 1), try rewriter.replaceAllUses(sum, one));
+    try rewriter.eraseInstruction(sum_instruction);
+
+    try std.testing.expect(module.values.get(sum) == null);
+    try std.testing.expect(module.instructions.get(sum_instruction) == null);
+
+    try validator.validate(&module);
+}
+
+test "Rewriter: add block parameter and sync branch calls" {
+    // shader compute @main
+    // {
+    //     %0: constant u32 = bits(0x1)
+    //
+    //     fn @main() -> void
+    //     {
+    //         .entry():
+    //             branch .merge()
+    //
+    //         .merge():
+    //             return
+    //
+    //         .alternate():
+    //             return
+    //     }
+    // }
+
+    var module = module_ir.Module.init(std.testing.allocator, .compute);
+    defer module.deinit();
+
+    var builder = Builder.init(&module);
+
+    const void_type = try builder.internType(.void);
+    const u32_type = try builder.internType(.{ .integer = .{ .bits = 32, .signedness = .unsigned } });
+
+    const one = try builder.internConstant(u32_type, .{ .integer_bits = 1 });
+
+    const main = try builder.addFunction(void_type, "main");
+    builder.setEntryPoint(main);
+
+    const entry = try builder.addBlock(main, "entry");
+    const merge = try builder.addBlock(main, "merge");
+    const alternate = try builder.addBlock(main, "alternate");
+
+    try builder.setTerminator(entry, .{ .branch = try builder.edge(merge, &.{}) });
+    try builder.setTerminator(merge, .return_void);
+    try builder.setTerminator(alternate, .return_void);
+
+    var rewriter = Self.init(&module);
+
+    const parameter = try rewriter.addBlockParameter(merge, u32_type, "incoming", &.{
+        .{
+            .predecessor = entry,
+            .value = one,
+        },
+    });
+    const merge_edge = module.blocks.get(entry).?.terminator.?.branch;
+    try std.testing.expectEqualSlices(ids.ValueId, &.{one}, merge_edge.arguments);
+
+    _ = try builder.appendInstruction(merge, u32_type, .{
+        .binary = .{
+            .opcode = .integer_add,
+            .lhs = parameter,
+            .rhs = one,
+        },
+    }, null);
+    try validator.validate(&module);
+
+    try rewriter.removeBlockParameter(merge, 0, one);
+
+    try std.testing.expectEqual(@as(usize, 0), module.blocks.get(merge).?.parameters.items.len);
+    try std.testing.expectEqual(@as(usize, 0), module.blocks.get(entry).?.terminator.?.branch.arguments.len);
+
+    try validator.validate(&module);
+
+    try std.testing.expectEqual(@as(usize, 1), try rewriter.redirectEdges(entry, merge, alternate, &.{}));
+    try std.testing.expectEqual(alternate, module.blocks.get(entry).?.terminator.?.branch.target);
+
+    try validator.validate(&module);
+}

src/compiler/ir/cfg.zig

@@ -19,8 +19,8 @@ reachable: []bool,
 dominators: []bool,
 
 pub fn init(allocator: std.mem.Allocator, module: *const module_ir.Module, function_id: ids.FunctionId) Error!Self {
-    const function = module.functions.get(function_id) orelse return error.InvalidFunction;
+    const function = module.functions.get(function_id) orelse return Error.InvalidFunction;
-    const entry = function.entry_block orelse return error.MissingEntryBlock;
+    const entry = function.entry_block orelse return Error.MissingEntryBlock;
     const blocks = try allocator.dupe(ids.BlockId, function.blocks.items);
     errdefer allocator.free(blocks);
 
@@ -84,8 +84,8 @@ pub fn dominates(self: *const Self, dominator: ids.BlockId, block: ids.BlockId)
 
 fn buildPredecessors(self: *Self, module: *const module_ir.Module) Error!void {
     for (self.blocks) |source| {
-        const block = module.blocks.get(source) orelse return error.InvalidBlock;
+        const block = module.blocks.get(source) orelse return Error.InvalidBlock;
-        const terminator = block.terminator orelse return error.MissingTerminator;
+        const terminator = block.terminator orelse return Error.MissingTerminator;
         switch (terminator) {
             .branch => |edge| try self.addPredecessor(edge.target, source),
             .conditional_branch => |branch| {
@@ -101,12 +101,12 @@ fn buildReachability(self: *Self, module: *const module_ir.Module, entry: ids.Bl
     var queue: std.ArrayList(ids.BlockId) = .empty;
     defer queue.deinit(self.allocator);
     try queue.append(self.allocator, entry);
-    self.reachable[self.indexOf(entry) orelse return error.InvalidBlock] = true;
+    self.reachable[self.indexOf(entry) orelse return Error.InvalidBlock] = true;
 
     var cursor: usize = 0;
     while (cursor < queue.items.len) : (cursor += 1) {
-        const block = module.blocks.get(queue.items[cursor]) orelse return error.InvalidBlock;
+        const block = module.blocks.get(queue.items[cursor]) orelse return Error.InvalidBlock;
-        const terminator = block.terminator orelse return error.MissingTerminator;
+        const terminator = block.terminator orelse return Error.MissingTerminator;
         switch (terminator) {
             .branch => |edge| try self.markReachable(&queue, edge.target),
             .conditional_branch => |branch| {
@@ -168,12 +168,12 @@ fn buildDominators(self: *Self, entry: ids.BlockId) void {
 }
 
 fn addPredecessor(self: *Self, target: ids.BlockId, source: ids.BlockId) Error!void {
-    const target_index = self.indexOf(target) orelse return error.CrossFunctionEdge;
+    const target_index = self.indexOf(target) orelse return Error.CrossFunctionEdge;
     try self.predecessors_by_block[target_index].append(self.allocator, source);
 }
 
 fn markReachable(self: *Self, queue: *std.ArrayList(ids.BlockId), target: ids.BlockId) Error!void {
-    const target_index = self.indexOf(target) orelse return error.CrossFunctionEdge;
+    const target_index = self.indexOf(target) orelse return Error.CrossFunctionEdge;
 
     if (self.reachable[target_index])
         return;

src/compiler/ir/instruction.zig

@@ -6,6 +6,7 @@ pub const ValueId = ids.ValueId;
 pub const BlockId = ids.BlockId;
 pub const FunctionId = ids.FunctionId;
 pub const InterfaceVariableId = ids.InterfaceVariableId;
+pub const ResourceId = ids.ResourceId;
 
 pub const SourceLocation = struct {
     file: ?[]const u8 = null,
@@ -98,11 +99,28 @@ pub const StoreInterface = struct {
     element_index: ?ValueId = null,
 };
 
+pub const LoadBuffer = struct {
+    resource: ResourceId,
+    byte_offset: ValueId,
+};
+
+pub const StoreBuffer = struct {
+    resource: ResourceId,
+    byte_offset: ValueId,
+    value: ValueId,
+};
+
 pub const Call = struct {
     function: FunctionId,
     arguments: []const ValueId,
 };
 
+pub const ArrayLength = struct {
+    resource: ResourceId,
+    byte_offset: ValueId,
+    stride: u32,
+};
+
 pub const Operation = union(enum) {
     unary: Unary,
     binary: Binary,
@@ -113,7 +131,10 @@ pub const Operation = union(enum) {
     composite_extract: CompositeExtract,
     load_interface: LoadInterface,
     store_interface: StoreInterface,
+    load_buffer: LoadBuffer,
+    store_buffer: StoreBuffer,
     call: Call,
+    array_length: ArrayLength,
 
     pub fn visitValueUses(self: Operation, context: anytype, comptime visitor: anytype) void {
         switch (self) {
@@ -140,10 +161,16 @@ pub const Operation = union(enum) {
                 if (op.element_index) |index|
                     visitor(context, index);
             },
+            .load_buffer => |op| visitor(context, op.byte_offset),
+            .store_buffer => |op| {
+                visitor(context, op.byte_offset);
+                visitor(context, op.value);
+            },
             .call => |op| {
                 for (op.arguments) |argument|
                     visitor(context, argument);
             },
+            .array_length => |op| visitor(context, op.byte_offset),
         }
     }
 
@@ -176,14 +203,24 @@ pub const Operation = union(enum) {
                 if (op.element_index) |*index|
                     replaceOne(index, old, replacement, &count);
             },
+            .load_buffer => |*op| replaceOne(&op.byte_offset, old, replacement, &count),
+            .store_buffer => |*op| {
+                replaceOne(&op.byte_offset, old, replacement, &count);
+                replaceOne(&op.value, old, replacement, &count);
+            },
             .call => |*op| op.arguments = try replaceSlice(allocator, op.arguments, old, replacement, &count),
+            .array_length => |*op| replaceOne(&op.byte_offset, old, replacement, &count),
         }
         return count;
     }
 
     pub fn hasSideEffects(self: Operation) bool {
         return switch (self) {
-            .store_interface, .call => true,
+            .store_interface,
+            .store_buffer,
+            .call,
+            => true,
+
             else => false,
         };
     }

src/compiler/ir/ir.zig

@@ -48,9 +48,10 @@ pub const cfg = @import("cfg.zig");
 pub const constant = @import("constant.zig");
 pub const id = @import("id.zig");
 pub const instruction = @import("instruction.zig");
+pub const inline_all_functions = @import("transformers/inline_all_functions.zig");
 pub const module = @import("module.zig");
 pub const parser = @import("parser/parser.zig");
-pub const pass_manager = @import("pass_manager.zig");
+pub const transformer_manager = @import("transformer_manager.zig");
 pub const printer = @import("printer.zig");
 pub const types = @import("type.zig");
 pub const validator = @import("validator/validator.zig");

src/compiler/ir/module.zig

@@ -94,6 +94,8 @@ pub const Builtin = enum {
     frag_coord,
     frag_depth,
     global_invocation_id,
+    num_workgroups,
+    workgroup_size,
 };
 
 pub const InterfaceSemantic = union(enum) {
@@ -233,3 +235,13 @@ fn replaceOne(operand: *ids.ValueId, old: ids.ValueId, replacement: ids.ValueId,
     operand.* = replacement;
     count.* += 1;
 }
+
+test "Module: central store IDs graveyard" {
+    var module = Module.init(std.testing.allocator, .fragment);
+    defer module.deinit();
+    const first = try module.internType(.boolean);
+    try std.testing.expect(module.types.remove(first));
+    const second = try module.internType(.boolean);
+    try std.testing.expect(first.index() != second.index());
+    try std.testing.expect(module.types.get(first) == null);
+}

src/compiler/ir/parser/ast.zig

@@ -2,12 +2,14 @@ const std = @import("std");
 const ids = @import("../id.zig");
 const inst_ir = @import("../instruction.zig");
 const module_ir = @import("../module.zig");
+const type_ir = @import("../type.zig");
 
 pub const ValueRef = []const u8;
 
 pub const ParsedModule = struct {
     entry_point_name: ?[]const u8,
     interfaces: std.ArrayList(ParsedInterface) = .empty,
+    resources: std.ArrayList(ParsedResource) = .empty,
     constants: std.ArrayList(ParsedConstant) = .empty,
     functions: std.ArrayList(ParsedFunction) = .empty,
 };
@@ -19,6 +21,14 @@ pub const ParsedInterface = struct {
     semantic: module_ir.InterfaceSemantic,
 };
 
+pub const ParsedResource = struct {
+    kind: type_ir.ResourceKind,
+    name: []const u8,
+    ty: ids.TypeId,
+    set: u32,
+    binding: u32,
+};
+
 pub const ParsedConstantValue = union(enum) {
     boolean: bool,
     integer_bits: u64,
@@ -89,5 +99,8 @@ pub const ParsedOperation = union(enum) {
     composite_extract: struct { composite: ValueRef, indices: []const u32 },
     load_interface: []const u8,
     store_interface: struct { interface_name: []const u8, value: ValueRef },
+    load_buffer: struct { resource_name: []const u8, byte_offset: ValueRef },
+    store_buffer: struct { resource_name: []const u8, byte_offset: ValueRef, value: ValueRef },
+    array_length: struct { resource_name: []const u8, byte_offset: ValueRef, stride: u32 },
     call: struct { function_name: []const u8, arguments: []const ValueRef },
 };

src/compiler/ir/parser/lower.zig

@@ -22,6 +22,7 @@ pub fn lower(allocator: std.mem.Allocator, module: *module_ir.Module, parsed: *P
     var values: std.StringHashMapUnmanaged(ids.ValueId) = .empty;
     var constants: std.AutoHashMapUnmanaged(u32, ids.ConstantId) = .empty;
     var interfaces: std.StringHashMapUnmanaged(ids.InterfaceVariableId) = .empty;
+    var resources: std.StringHashMapUnmanaged(ids.ResourceId) = .empty;
     var functions: std.StringHashMapUnmanaged(ids.FunctionId) = .empty;
 
     for (parsed.interfaces.items) |interface| {
@@ -32,6 +33,14 @@ pub fn lower(allocator: std.mem.Allocator, module: *module_ir.Module, parsed: *P
         try interfaces.put(allocator, interface.name, id);
     }
 
+    for (parsed.resources.items) |resource| {
+        if (resources.contains(resource.name))
+            return error.DuplicateName;
+
+        const id = try builder.addResource(resource.ty, resource.kind, resource.set, resource.binding, resource.name);
+        try resources.put(allocator, resource.name, id);
+    }
+
     for (parsed.constants.items, 0..) |constant, constant_index| {
         const value: constant_ir.ConstantValue = switch (constant.value) {
             .boolean => |item| .{ .boolean = item },
@@ -94,7 +103,7 @@ pub fn lower(allocator: std.mem.Allocator, module: *module_ir.Module, parsed: *P
 
         for (function.blocks.items) |block| {
             for (block.instructions.items) |instruction| {
-                const lowered = try lowerOperation(allocator, module, &values, &interfaces, &functions, instruction.operation);
+                const lowered = try lowerOperation(allocator, module, &values, &interfaces, &resources, &functions, instruction.operation);
                 const result_type = instruction.result_type orelse lowered.inferred_type;
 
                 if (instruction.printed_result != null and result_type == null)
@@ -125,6 +134,7 @@ fn lowerOperation(
     module: *module_ir.Module,
     values: *const std.StringHashMapUnmanaged(ids.ValueId),
     interfaces: *const std.StringHashMapUnmanaged(ids.InterfaceVariableId),
+    resources: *const std.StringHashMapUnmanaged(ids.ResourceId),
     functions: *const std.StringHashMapUnmanaged(ids.FunctionId),
     parsed: ParsedOperation,
 ) !LoweredOperation {
@@ -249,6 +259,45 @@ fn lowerOperation(
                 .inferred_type = null,
             };
         },
+        .load_buffer => |op| blk: {
+            const resource_id = resources.get(op.resource_name) orelse return error.UnknownResource;
+            const byte_offset = resolveValue(values, op.byte_offset) orelse return error.UnknownValue;
+
+            break :blk .{
+                .operation = .{ .load_buffer = .{
+                    .resource = resource_id,
+                    .byte_offset = byte_offset,
+                } },
+                .inferred_type = null,
+            };
+        },
+        .store_buffer => |op| blk: {
+            const resource_id = resources.get(op.resource_name) orelse return error.UnknownResource;
+            const byte_offset = resolveValue(values, op.byte_offset) orelse return error.UnknownValue;
+            const value = resolveValue(values, op.value) orelse return error.UnknownValue;
+
+            break :blk .{
+                .operation = .{ .store_buffer = .{
+                    .resource = resource_id,
+                    .byte_offset = byte_offset,
+                    .value = value,
+                } },
+                .inferred_type = null,
+            };
+        },
+        .array_length => |op| blk: {
+            const resource_id = resources.get(op.resource_name) orelse return error.UnknownResource;
+            const byte_offset = resolveValue(values, op.byte_offset) orelse return error.UnknownValue;
+
+            break :blk .{
+                .operation = .{ .array_length = .{
+                    .resource = resource_id,
+                    .byte_offset = byte_offset,
+                    .stride = op.stride,
+                } },
+                .inferred_type = null,
+            };
+        },
         .call => |op| blk: {
             const function_id = functions.get(op.function_name) orelse return error.UnknownFunction;
             var arguments: std.ArrayList(ids.ValueId) = .empty;

src/compiler/ir/pass_manager.zig

@@ -1,83 +0,0 @@
-const std = @import("std");
-const module_ir = @import("module.zig");
-const validator = @import("validator/validator.zig");
-
-pub const Context = struct {
-    allocator: std.mem.Allocator,
-    validate_after_each_pass: bool = true,
-};
-
-pub const Pass = struct {
-    name: []const u8,
-    required: module_ir.Properties = .{},
-    produced: module_ir.Properties = .{},
-    invalidated: module_ir.Properties = .{},
-    run: *const fn (module: *module_ir.Module, context: *Context) anyerror!bool,
-};
-
-pub const Manager = struct {
-    allocator: std.mem.Allocator,
-    passes: std.ArrayList(Pass) = .empty,
-
-    pub fn init(allocator: std.mem.Allocator) Manager {
-        return .{ .allocator = allocator };
-    }
-
-    pub fn deinit(self: *Manager) void {
-        self.passes.deinit(self.allocator);
-        self.* = undefined;
-    }
-
-    pub fn add(self: *Manager, pass: Pass) !void {
-        try self.passes.append(self.allocator, pass);
-    }
-
-    pub fn run(self: *Manager, module: *module_ir.Module, context: *Context) !bool {
-        var changed = false;
-        for (self.passes.items) |pass| {
-            if (!satisfies(module.properties, pass.required))
-                return error.RequiredPropertyMissing;
-
-            changed = (try pass.run(module, context)) or changed;
-            applyInvalidated(&module.properties, pass.invalidated);
-            applyProduced(&module.properties, pass.produced);
-
-            if (context.validate_after_each_pass)
-                try validator.validate(module);
-        }
-        return changed;
-    }
-};
-
-fn satisfies(actual: module_ir.Properties, required: module_ir.Properties) bool {
-    inline for (property_names) |name| {
-        if (@field(required, name) and !@field(actual, name))
-            return false;
-    }
-    return true;
-}
-
-fn applyProduced(properties: *module_ir.Properties, produced: module_ir.Properties) void {
-    inline for (property_names) |name| {
-        if (@field(produced, name))
-            @field(properties, name) = true;
-    }
-}
-
-fn applyInvalidated(properties: *module_ir.Properties, invalidated: module_ir.Properties) void {
-    inline for (property_names) |name| {
-        if (@field(invalidated, name))
-            @field(properties, name) = false;
-    }
-}
-
-const property_names = .{
-    "valid_cfg",
-    "valid_ssa",
-    "structured_control_flow",
-    "no_function_calls",
-    "no_local_memory",
-    "no_matrix_types",
-    "no_large_composites",
-    "explicit_resource_offsets",
-};

src/compiler/ir/printer.zig

@@ -32,6 +32,16 @@ pub fn write(module: *const module_ir.Module, writer: *std.Io.Writer) std.Io.Wri
         try writer.writeAll("]\n");
     }
 
+    for (module.resources.entries.items, 0..) |entry, index| {
+        const resource = entry orelse continue;
+
+        try writer.writeAll(indent);
+        try writeNamedRef(writer, resource.name, "resource", index);
+        try writer.writeAll(": ");
+        try writeType(module, writer, resource.type);
+        try writer.print(" = {t}[set({d}), binding({d})]\n", .{ resource.kind, resource.set, resource.binding });
+    }
+
     for (module.constants.entries.items, 0..) |entry, constant_index| {
         const constant = entry orelse continue;
         const value_id = constantValueId(module, ids.ConstantId.fromIndex(constant_index)) orelse continue;
@@ -160,6 +170,11 @@ fn writeType(module: *const module_ir.Module, writer: *std.Io.Writer, type_id: i
             try writer.writeByte(']');
         },
         .resource_handle => |handle| try writer.print("resourceHandle[{t}]", .{handle.kind}),
+        .runtime_array => |array| {
+            try writer.writeAll("runtime_array[");
+            try writeType(module, writer, array.element_type);
+            try writer.writeByte(']');
+        },
     }
 }
 
@@ -214,6 +229,22 @@ fn writeOperation(module: *const module_ir.Module, writer: *std.Io.Writer, opera
             try writer.writeAll(", ");
             try writeValueRef(module, writer, op.value);
         },
+        .load_buffer => |op| {
+            try writer.writeAll("load_buffer ");
+            const resource = module.resources.get(op.resource);
+            try writeNamedRef(writer, if (resource) |r| r.name else null, "resource", op.resource.index());
+            try writer.writeAll(", ");
+            try writeValueRef(module, writer, op.byte_offset);
+        },
+        .store_buffer => |op| {
+            try writer.writeAll("store_buffer ");
+            const resource = module.resources.get(op.resource);
+            try writeNamedRef(writer, if (resource) |r| r.name else null, "resource", op.resource.index());
+            try writer.writeAll(", ");
+            try writeValueRef(module, writer, op.byte_offset);
+            try writer.writeAll(", ");
+            try writeValueRef(module, writer, op.value);
+        },
         .call => |op| {
             try writer.writeAll("call ");
             try writeFunctionRef(module, writer, op.function);
@@ -221,6 +252,14 @@ fn writeOperation(module: *const module_ir.Module, writer: *std.Io.Writer, opera
             try writeValueList(module, writer, op.arguments);
             try writer.writeByte(')');
         },
+        .array_length => |op| {
+            try writer.writeAll("array_length ");
+            const resource = module.resources.get(op.resource);
+            try writeNamedRef(writer, if (resource) |r| r.name else null, "resource", op.resource.index());
+            try writer.writeAll(", ");
+            try writeValueRef(module, writer, op.byte_offset);
+            try writer.print(", stride {}", .{op.stride});
+        },
     }
 }
 

src/compiler/ir/transformer_manager.zig

@@ -0,0 +1,157 @@
+const std = @import("std");
+const ids = @import("id.zig");
+const module_ir = @import("module.zig");
+const Builder = @import("Builder.zig");
+const validator = @import("validator/validator.zig");
+const visitor = @import("visitor.zig");
+
+pub const Context = struct {
+    allocator: std.mem.Allocator,
+    validate_after_each_transformer: bool = true,
+};
+
+pub const Transformer = struct {
+    name: []const u8,
+    required: module_ir.Properties = .{},
+    produced: module_ir.Properties = .{},
+    invalidated: module_ir.Properties = .{},
+    run: *const fn (module: *module_ir.Module, context: *Context) anyerror!bool,
+};
+
+pub const Manager = struct {
+    allocator: std.mem.Allocator,
+    transformers: std.ArrayList(Transformer) = .empty,
+
+    pub fn init(allocator: std.mem.Allocator) Manager {
+        return .{ .allocator = allocator };
+    }
+
+    pub fn deinit(self: *Manager) void {
+        self.transformers.deinit(self.allocator);
+        self.* = undefined;
+    }
+
+    pub fn add(self: *Manager, transformer: Transformer) !void {
+        try self.transformers.append(self.allocator, transformer);
+    }
+
+    pub fn run(self: *Manager, module: *module_ir.Module, context: *Context) !bool {
+        var changed = false;
+        for (self.transformers.items) |transformer| {
+            if (!satisfies(module.properties, transformer.required))
+                return error.RequiredPropertyMissing;
+
+            changed = (try transformer.run(module, context)) or changed;
+            applyInvalidated(&module.properties, transformer.invalidated);
+            applyProduced(&module.properties, transformer.produced);
+
+            if (context.validate_after_each_transformer)
+                try validator.validate(module);
+        }
+        return changed;
+    }
+};
+
+fn satisfies(actual: module_ir.Properties, required: module_ir.Properties) bool {
+    inline for (property_names) |name| {
+        if (@field(required, name) and !@field(actual, name))
+            return false;
+    }
+    return true;
+}
+
+fn applyProduced(properties: *module_ir.Properties, produced: module_ir.Properties) void {
+    inline for (property_names) |name| {
+        if (@field(produced, name))
+            @field(properties, name) = true;
+    }
+}
+
+fn applyInvalidated(properties: *module_ir.Properties, invalidated: module_ir.Properties) void {
+    inline for (property_names) |name| {
+        if (@field(invalidated, name))
+            @field(properties, name) = false;
+    }
+}
+
+const property_names = .{
+    "valid_cfg",
+    "valid_ssa",
+    "structured_control_flow",
+    "no_function_calls",
+    "no_local_memory",
+    "no_matrix_types",
+    "no_large_composites",
+    "explicit_resource_offsets",
+};
+
+const VisitorStatistics = struct {
+    functions: usize = 0,
+    blocks: usize = 0,
+};
+
+fn establishNoCalls(_: *module_ir.Module, _: *Context) !bool {
+    return false;
+}
+
+fn countVisitedFunction(context: ?*anyopaque, _: ids.FunctionId, _: *const module_ir.Function) !void {
+    const statistics: *VisitorStatistics = @ptrCast(@alignCast(context.?));
+    statistics.functions += 1;
+}
+
+fn countVisitedBlock(context: ?*anyopaque, _: ids.BlockId, _: *const module_ir.Block) !void {
+    const statistics: *VisitorStatistics = @ptrCast(@alignCast(context.?));
+    statistics.blocks += 1;
+}
+
+test "Transformers manager: tracks independent IR properties" {
+    // shader compute @main
+    // {
+    //     fn @main() -> void
+    //     {
+    //         .entry():
+    //             return
+    //     }
+    // }
+
+    var module = module_ir.Module.init(std.testing.allocator, .compute);
+    defer module.deinit();
+
+    var builder = Builder.init(&module);
+
+    const void_type = try builder.internType(.void);
+
+    const main = try builder.addFunction(void_type, "main");
+    builder.setEntryPoint(main);
+
+    const entry = try builder.addBlock(main, "entry");
+    try builder.setTerminator(entry, .return_void);
+
+    module.properties.valid_cfg = true;
+
+    var manager = Manager.init(std.testing.allocator);
+    defer manager.deinit();
+
+    try manager.add(.{
+        .name = "establish-no-calls",
+        .required = .{ .valid_cfg = true },
+        .produced = .{ .no_function_calls = true },
+        .run = establishNoCalls,
+    });
+
+    var context: Context = .{ .allocator = std.testing.allocator };
+
+    try std.testing.expect(!try manager.run(&module, &context));
+    try std.testing.expect(module.properties.no_function_calls);
+
+    var statistics: VisitorStatistics = .{};
+
+    try visitor.walk(&module, .{
+        .context = &statistics,
+        .visitFunction = countVisitedFunction,
+        .visitBlock = countVisitedBlock,
+    });
+
+    try std.testing.expectEqual(@as(usize, 1), statistics.functions);
+    try std.testing.expectEqual(@as(usize, 1), statistics.blocks);
+}

src/compiler/ir/transformers/inline_all_functions.zig

@@ -0,0 +1,765 @@
+const std = @import("std");
+const Builder = @import("../Builder.zig");
+const Rewriter = @import("../Rewriter.zig");
+const ids = @import("../id.zig");
+const instruction_ir = @import("../instruction.zig");
+const module_ir = @import("../module.zig");
+const transformer_manager = @import("../transformer_manager.zig");
+
+pub const Error = Rewriter.Error || error{
+    InvalidModule,
+    RecursiveCall,
+    UnsupportedEntryBlockParameters,
+};
+
+const VisitState = enum {
+    unvisited,
+    visiting,
+    complete,
+};
+
+const CallGraph = struct {
+    module: *const module_ir.Module,
+    states: []VisitState,
+    postorder: *std.ArrayList(ids.FunctionId),
+    allocator: std.mem.Allocator,
+
+    fn visit(self: *CallGraph, function_id: ids.FunctionId) Error!void {
+        if (function_id.index() >= self.states.len)
+            return Error.InvalidModule;
+
+        switch (self.states[function_id.index()]) {
+            .visiting => return Error.RecursiveCall,
+            .complete => return,
+            .unvisited => {},
+        }
+        self.states[function_id.index()] = .visiting;
+
+        const function = self.module.functions.get(function_id) orelse return Error.InvalidModule;
+        for (function.blocks.items) |block_id| {
+            const block = self.module.blocks.get(block_id) orelse return Error.InvalidModule;
+            for (block.instructions.items) |instruction_id| {
+                const inst = self.module.instructions.get(instruction_id) orelse return Error.InvalidModule;
+                switch (inst.operation) {
+                    .call => |call| try self.visit(call.function),
+                    else => {},
+                }
+            }
+        }
+
+        self.states[function_id.index()] = .complete;
+        try self.postorder.append(self.allocator, function_id);
+    }
+};
+
+pub const transformer: transformer_manager.Transformer = .{
+    .name = "inline-all-functions",
+    .produced = .{ .no_function_calls = true },
+    .invalidated = .{ .structured_control_flow = true },
+    .run = transform,
+};
+
+fn transform(module: *module_ir.Module, context: *transformer_manager.Context) !bool {
+    const scratch_allocator = context.allocator;
+    const entry_point = module.entry_point orelse return Error.InvalidModule;
+    if (!module.functions.isLive(entry_point))
+        return Error.InvalidModule;
+
+    var changed = try removeUnreachableBlocks(module, scratch_allocator);
+
+    const states = try scratch_allocator.alloc(VisitState, module.functions.entries.items.len);
+    defer scratch_allocator.free(states);
+    @memset(states, .unvisited);
+
+    var postorder: std.ArrayList(ids.FunctionId) = .empty;
+    defer postorder.deinit(scratch_allocator);
+
+    var call_graph: CallGraph = .{
+        .module = module,
+        .states = states,
+        .postorder = &postorder,
+        .allocator = scratch_allocator,
+    };
+    try call_graph.visit(entry_point);
+
+    for (postorder.items) |function_id|
+        changed = (try inlineCallsInFunction(module, scratch_allocator, function_id)) or changed;
+
+    changed = removeNonEntryFunctions(module, entry_point) or changed;
+
+    for (module.instructions.entries.items) |entry| {
+        const inst = entry orelse continue;
+        if (inst.operation == .call)
+            return Error.InvalidModule;
+    }
+
+    return changed;
+}
+
+fn inlineCallsInFunction(module: *module_ir.Module, scratch_allocator: std.mem.Allocator, function_id: ids.FunctionId) Error!bool {
+    const function = module.functions.get(function_id) orelse return Error.InvalidModule;
+    var calls: std.ArrayList(ids.InstructionId) = .empty;
+    defer calls.deinit(scratch_allocator);
+
+    for (function.blocks.items) |block_id| {
+        const block = module.blocks.get(block_id) orelse return Error.InvalidModule;
+        for (block.instructions.items) |instruction_id| {
+            const inst = module.instructions.get(instruction_id) orelse return Error.InvalidModule;
+            if (inst.operation == .call)
+                try calls.append(scratch_allocator, instruction_id);
+        }
+    }
+
+    for (calls.items) |call_id|
+        try inlineCall(module, scratch_allocator, function_id, call_id);
+
+    return calls.items.len != 0;
+}
+
+fn inlineCall(module: *module_ir.Module, scratch_allocator: std.mem.Allocator, caller_id: ids.FunctionId, call_id: ids.InstructionId) Error!void {
+    const call_inst = module.instructions.get(call_id) orelse return Error.InvalidModule;
+    if (call_inst.operation != .call)
+        return Error.InvalidModule;
+
+    const call = call_inst.operation.call;
+    if (call.function == caller_id)
+        return Error.RecursiveCall;
+
+    const callee = module.functions.get(call.function) orelse return Error.InvalidModule;
+    const callee_entry = module.blocks.get(callee.entry_block orelse return Error.InvalidModule) orelse return Error.InvalidModule;
+    if (callee_entry.parameters.items.len != 0)
+        return Error.UnsupportedEntryBlockParameters;
+
+    const caller_block_id = call_inst.parent_block;
+    const caller_block = module.blocks.get(caller_block_id) orelse return Error.InvalidModule;
+    if (caller_block.parent_function != caller_id)
+        return Error.InvalidModule;
+
+    var call_index: ?usize = null;
+    for (caller_block.instructions.items, 0..) |instruction_id, index| {
+        if (instruction_id == call_id) {
+            call_index = index;
+            break;
+        }
+    }
+    const index = call_index orelse return Error.InvalidModule;
+
+    const arguments = try scratch_allocator.dupe(ids.ValueId, call.arguments);
+    defer scratch_allocator.free(arguments);
+
+    const suffix = try scratch_allocator.dupe(ids.InstructionId, caller_block.instructions.items[index + 1 ..]);
+    defer scratch_allocator.free(suffix);
+
+    const old_terminator = caller_block.terminator orelse return Error.InvalidModule;
+    const old_structured_control = caller_block.structured_control;
+    const call_result = call_inst.result;
+    const result_type = if (call_result) |result_id|
+        (module.values.get(result_id) orelse return Error.InvalidModule).type
+    else
+        null;
+    const result_name = if (call_result) |result_id|
+        (module.values.get(result_id) orelse return Error.InvalidModule).name
+    else
+        null;
+
+    var builder = Builder.init(module);
+    const continuation_id = try builder.addBlock(caller_id, null);
+    const continuation_result = if (result_type) |ty|
+        try builder.addBlockParameter(continuation_id, ty, result_name)
+    else
+        null;
+
+    {
+        const continuation = module.blocks.getMut(continuation_id) orelse return Error.InvalidModule;
+        try continuation.instructions.appendSlice(module.allocator(), suffix);
+        continuation.terminator = old_terminator;
+        continuation.structured_control = switch (old_structured_control) {
+            .loop => .none,
+            else => old_structured_control,
+        };
+    }
+    for (suffix) |instruction_id| {
+        const moved = module.instructions.getMut(instruction_id) orelse return Error.InvalidModule;
+        moved.parent_block = continuation_id;
+    }
+
+    {
+        const mutable_caller_block = module.blocks.getMut(caller_block_id) orelse return Error.InvalidModule;
+        mutable_caller_block.instructions.shrinkRetainingCapacity(index);
+        mutable_caller_block.terminator = null;
+        mutable_caller_block.structured_control = switch (old_structured_control) {
+            .loop => old_structured_control,
+            else => .none,
+        };
+    }
+
+    if (call_result) |old_result| {
+        const replacement = continuation_result orelse return Error.InvalidModule;
+        var rewriter = Rewriter.init(module);
+        _ = try rewriter.replaceAllUses(old_result, replacement);
+    }
+
+    const cloned_entry = try cloneCallee(
+        module,
+        scratch_allocator,
+        caller_id,
+        call.function,
+        arguments,
+        continuation_id,
+        continuation_result != null,
+    );
+
+    const cloned_entry_block = module.blocks.get(cloned_entry) orelse return Error.InvalidModule;
+    if (cloned_entry_block.parameters.items.len != 0)
+        return Error.UnsupportedEntryBlockParameters;
+
+    const branch_arguments = try module.allocator().alloc(ids.ValueId, 0);
+    const mutable_caller_block = module.blocks.getMut(caller_block_id) orelse return Error.InvalidModule;
+    mutable_caller_block.terminator = .{ .branch = .{
+        .target = cloned_entry,
+        .arguments = branch_arguments,
+    } };
+
+    if (call_result) |result_id|
+        _ = module.values.remove(result_id);
+    _ = module.instructions.remove(call_id);
+}
+
+fn cloneCallee(
+    module: *module_ir.Module,
+    scratch_allocator: std.mem.Allocator,
+    caller_id: ids.FunctionId,
+    callee_id: ids.FunctionId,
+    arguments: []const ids.ValueId,
+    continuation_id: ids.BlockId,
+    returns_value: bool,
+) Error!ids.BlockId {
+    const callee = module.functions.get(callee_id) orelse return Error.InvalidModule;
+    if (callee.parameters.items.len != arguments.len)
+        return Error.InvalidModule;
+
+    const callee_blocks = try scratch_allocator.dupe(ids.BlockId, callee.blocks.items);
+    defer scratch_allocator.free(callee_blocks);
+    const callee_entry = callee.entry_block orelse return Error.InvalidModule;
+
+    const block_map = try scratch_allocator.alloc(?ids.BlockId, module.blocks.entries.items.len);
+    defer scratch_allocator.free(block_map);
+    @memset(block_map, null);
+
+    const value_map = try scratch_allocator.alloc(?ids.ValueId, module.values.entries.items.len);
+    defer scratch_allocator.free(value_map);
+    @memset(value_map, null);
+
+    const instruction_map = try scratch_allocator.alloc(?ids.InstructionId, module.instructions.entries.items.len);
+    defer scratch_allocator.free(instruction_map);
+    @memset(instruction_map, null);
+
+    for (callee.parameters.items, arguments) |parameter, argument| {
+        if (parameter.index() >= value_map.len)
+            return Error.InvalidModule;
+        value_map[parameter.index()] = argument;
+    }
+
+    var builder = Builder.init(module);
+    for (callee_blocks) |old_block_id| {
+        const new_block_id = try builder.addBlock(caller_id, null);
+        if (old_block_id.index() >= block_map.len)
+            return Error.InvalidModule;
+        block_map[old_block_id.index()] = new_block_id;
+    }
+
+    for (callee_blocks) |old_block_id| {
+        const old_block = module.blocks.get(old_block_id) orelse return Error.InvalidModule;
+        const new_block_id = mappedBlock(block_map, old_block_id) orelse return Error.InvalidModule;
+        for (old_block.parameters.items) |old_parameter| {
+            const old_value = module.values.get(old_parameter) orelse return Error.InvalidModule;
+            const new_parameter = try builder.addBlockParameter(new_block_id, old_value.type, null);
+            if (old_parameter.index() >= value_map.len)
+                return Error.InvalidModule;
+            value_map[old_parameter.index()] = new_parameter;
+        }
+    }
+
+    for (callee_blocks) |old_block_id| {
+        const old_block = module.blocks.get(old_block_id) orelse return Error.InvalidModule;
+        const new_block_id = mappedBlock(block_map, old_block_id) orelse return Error.InvalidModule;
+
+        for (old_block.instructions.items) |old_instruction_id| {
+            const old_instruction = (module.instructions.get(old_instruction_id) orelse return Error.InvalidModule).*;
+            const new_instruction_id = try module.instructions.add(module.allocator(), .{
+                .parent_block = new_block_id,
+                .result = null,
+                .operation = old_instruction.operation,
+                .source = old_instruction.source,
+            });
+            if (old_instruction_id.index() >= instruction_map.len)
+                return Error.InvalidModule;
+            instruction_map[old_instruction_id.index()] = new_instruction_id;
+
+            if (old_instruction.result) |old_result| {
+                const old_value = module.values.get(old_result) orelse return Error.InvalidModule;
+                const new_result = try module.values.add(module.allocator(), .{
+                    .type = old_value.type,
+                    .definition = .{ .instruction = new_instruction_id },
+                    .name = null,
+                });
+                module.instructions.getMut(new_instruction_id).?.result = new_result;
+                if (old_result.index() >= value_map.len)
+                    return Error.InvalidModule;
+                value_map[old_result.index()] = new_result;
+            }
+
+            const new_block = module.blocks.getMut(new_block_id) orelse return Error.InvalidModule;
+            try new_block.instructions.append(module.allocator(), new_instruction_id);
+        }
+    }
+
+    for (callee_blocks) |old_block_id| {
+        const old_block = module.blocks.get(old_block_id) orelse return Error.InvalidModule;
+        const new_block_id = mappedBlock(block_map, old_block_id) orelse return Error.InvalidModule;
+
+        for (old_block.instructions.items) |old_instruction_id| {
+            const new_instruction_id = mappedInstruction(instruction_map, old_instruction_id) orelse return Error.InvalidModule;
+            const new_instruction = module.instructions.getMut(new_instruction_id) orelse return Error.InvalidModule;
+            new_instruction.operation = try remapOperation(module, value_map, new_instruction.operation);
+        }
+
+        const new_block = module.blocks.getMut(new_block_id) orelse return Error.InvalidModule;
+        new_block.structured_control = try remapStructuredControl(block_map, old_block.structured_control);
+        new_block.terminator = try remapTerminator(
+            module,
+            value_map,
+            block_map,
+            old_block.terminator orelse return Error.InvalidModule,
+            continuation_id,
+            returns_value,
+        );
+    }
+
+    return mappedBlock(block_map, callee_entry) orelse return Error.InvalidModule;
+}
+
+fn remapOperation(
+    module: *module_ir.Module,
+    value_map: []const ?ids.ValueId,
+    operation: instruction_ir.Operation,
+) Error!instruction_ir.Operation {
+    return switch (operation) {
+        .unary => |op| .{ .unary = .{
+            .opcode = op.opcode,
+            .operand = try mappedValue(module, value_map, op.operand),
+        } },
+        .binary => |op| .{ .binary = .{
+            .opcode = op.opcode,
+            .lhs = try mappedValue(module, value_map, op.lhs),
+            .rhs = try mappedValue(module, value_map, op.rhs),
+        } },
+        .compare => |op| .{ .compare = .{
+            .opcode = op.opcode,
+            .lhs = try mappedValue(module, value_map, op.lhs),
+            .rhs = try mappedValue(module, value_map, op.rhs),
+        } },
+        .select => |op| .{ .select = .{
+            .condition = try mappedValue(module, value_map, op.condition),
+            .true_value = try mappedValue(module, value_map, op.true_value),
+            .false_value = try mappedValue(module, value_map, op.false_value),
+        } },
+        .bitcast => |operand| .{ .bitcast = try mappedValue(module, value_map, operand) },
+        .composite_construct => |op| .{ .composite_construct = .{
+            .elements = try remapValues(module, value_map, op.elements),
+        } },
+        .composite_extract => |op| .{ .composite_extract = .{
+            .composite = try mappedValue(module, value_map, op.composite),
+            .indices = op.indices,
+        } },
+        .load_interface => |op| .{ .load_interface = .{
+            .variable = op.variable,
+            .element_index = if (op.element_index) |index| try mappedValue(module, value_map, index) else null,
+        } },
+        .store_interface => |op| .{ .store_interface = .{
+            .variable = op.variable,
+            .value = try mappedValue(module, value_map, op.value),
+            .element_index = if (op.element_index) |index| try mappedValue(module, value_map, index) else null,
+        } },
+        .load_buffer => |op| .{ .load_buffer = .{
+            .resource = op.resource,
+            .byte_offset = try mappedValue(module, value_map, op.byte_offset),
+        } },
+        .store_buffer => |op| .{ .store_buffer = .{
+            .resource = op.resource,
+            .byte_offset = try mappedValue(module, value_map, op.byte_offset),
+            .value = try mappedValue(module, value_map, op.value),
+        } },
+        .call => Error.InvalidModule,
+        .array_length => |op| .{ .array_length = .{
+            .resource = op.resource,
+            .byte_offset = try mappedValue(module, value_map, op.byte_offset),
+            .stride = op.stride,
+        } },
+    };
+}
+
+fn remapTerminator(
+    module: *module_ir.Module,
+    value_map: []const ?ids.ValueId,
+    block_map: []const ?ids.BlockId,
+    terminator: module_ir.Terminator,
+    continuation_id: ids.BlockId,
+    returns_value: bool,
+) Error!module_ir.Terminator {
+    return switch (terminator) {
+        .branch => |edge| .{ .branch = try remapEdge(module, value_map, block_map, edge) },
+        .conditional_branch => |branch| .{ .conditional_branch = .{
+            .condition = try mappedValue(module, value_map, branch.condition),
+            .true_edge = try remapEdge(module, value_map, block_map, branch.true_edge),
+            .false_edge = try remapEdge(module, value_map, block_map, branch.false_edge),
+        } },
+        .return_void => if (returns_value)
+            Error.InvalidModule
+        else
+            .{ .branch = .{
+                .target = continuation_id,
+                .arguments = try module.allocator().alloc(ids.ValueId, 0),
+            } },
+        .return_value => |value| if (!returns_value)
+            Error.InvalidModule
+        else
+            .{ .branch = .{
+                .target = continuation_id,
+                .arguments = try remapValues(module, value_map, &.{value}),
+            } },
+        .discard => .discard,
+        .@"unreachable" => .@"unreachable",
+    };
+}
+
+fn remapEdge(module: *module_ir.Module, value_map: []const ?ids.ValueId, block_map: []const ?ids.BlockId, edge: module_ir.Edge) Error!module_ir.Edge {
+    return .{
+        .target = mappedBlock(block_map, edge.target) orelse return Error.InvalidModule,
+        .arguments = try remapValues(module, value_map, edge.arguments),
+    };
+}
+
+fn remapStructuredControl(block_map: []const ?ids.BlockId, control: module_ir.StructuredControl) Error!module_ir.StructuredControl {
+    return switch (control) {
+        .none => .none,
+        .selection => |selection| .{ .selection = .{
+            .merge_block = mappedBlock(block_map, selection.merge_block) orelse return Error.InvalidModule,
+        } },
+        .loop => |loop| .{ .loop = .{
+            .merge_block = mappedBlock(block_map, loop.merge_block) orelse return Error.InvalidModule,
+            .continue_block = mappedBlock(block_map, loop.continue_block) orelse return Error.InvalidModule,
+        } },
+    };
+}
+
+fn remapValues(module: *module_ir.Module, value_map: []const ?ids.ValueId, values: []const ids.ValueId) Error![]const ids.ValueId {
+    const result = try module.allocator().alloc(ids.ValueId, values.len);
+    for (values, result) |value, *mapped|
+        mapped.* = try mappedValue(module, value_map, value);
+    return result;
+}
+
+fn mappedValue(module: *const module_ir.Module, value_map: []const ?ids.ValueId, value_id: ids.ValueId) Error!ids.ValueId {
+    if (value_id.index() < value_map.len) {
+        if (value_map[value_id.index()]) |mapped|
+            return mapped;
+    }
+
+    const value = module.values.get(value_id) orelse return Error.InvalidModule;
+    return switch (value.definition) {
+        .constant, .undef => value_id,
+        else => Error.InvalidModule,
+    };
+}
+
+fn removeUnreachableBlocks(module: *module_ir.Module, scratch_allocator: std.mem.Allocator) Error!bool {
+    const reachable = try scratch_allocator.alloc(bool, module.blocks.entries.items.len);
+    defer scratch_allocator.free(reachable);
+
+    var queue: std.ArrayList(ids.BlockId) = .empty;
+    defer queue.deinit(scratch_allocator);
+
+    var changed = false;
+    for (module.functions.entries.items, 0..) |entry, function_index| {
+        const function = entry orelse continue;
+        const function_id = ids.FunctionId.fromIndex(function_index);
+        const entry_block = function.entry_block orelse return Error.InvalidModule;
+
+        @memset(reachable, false);
+        queue.clearRetainingCapacity();
+        try enqueueReachable(module, scratch_allocator, function_id, reachable, &queue, entry_block);
+
+        var cursor: usize = 0;
+        while (cursor < queue.items.len) : (cursor += 1) {
+            const block = module.blocks.get(queue.items[cursor]) orelse return Error.InvalidModule;
+            switch (block.terminator orelse return Error.InvalidModule) {
+                .branch => |edge| try enqueueReachable(module, scratch_allocator, function_id, reachable, &queue, edge.target),
+                .conditional_branch => |branch| {
+                    try enqueueReachable(module, scratch_allocator, function_id, reachable, &queue, branch.true_edge.target);
+                    try enqueueReachable(module, scratch_allocator, function_id, reachable, &queue, branch.false_edge.target);
+                },
+                else => {},
+            }
+        }
+
+        for (queue.items) |block_id| {
+            const block = module.blocks.get(block_id) orelse return Error.InvalidModule;
+            switch (block.structured_control) {
+                .none => {},
+                .selection => |selection| if (!isReachable(reachable, selection.merge_block))
+                    return Error.InvalidModule,
+                .loop => |loop| if (!isReachable(reachable, loop.merge_block) or !isReachable(reachable, loop.continue_block))
+                    return Error.InvalidModule,
+            }
+        }
+
+        const mutable_function = module.functions.getMut(function_id) orelse return Error.InvalidModule;
+        var block_index: usize = 0;
+        while (block_index < mutable_function.blocks.items.len) {
+            const block_id = mutable_function.blocks.items[block_index];
+            if (isReachable(reachable, block_id)) {
+                block_index += 1;
+                continue;
+            }
+
+            removeBlock(module, block_id);
+            _ = mutable_function.blocks.orderedRemove(block_index);
+            changed = true;
+        }
+    }
+
+    return changed;
+}
+
+fn enqueueReachable(
+    module: *const module_ir.Module,
+    scratch_allocator: std.mem.Allocator,
+    function_id: ids.FunctionId,
+    reachable: []bool,
+    queue: *std.ArrayList(ids.BlockId),
+    block_id: ids.BlockId,
+) Error!void {
+    if (block_id.index() >= reachable.len)
+        return Error.InvalidModule;
+    if (reachable[block_id.index()])
+        return;
+
+    const block = module.blocks.get(block_id) orelse return Error.InvalidModule;
+    if (block.parent_function != function_id)
+        return Error.InvalidModule;
+
+    reachable[block_id.index()] = true;
+    try queue.append(scratch_allocator, block_id);
+}
+
+fn isReachable(reachable: []const bool, block_id: ids.BlockId) bool {
+    return block_id.index() < reachable.len and reachable[block_id.index()];
+}
+
+fn removeBlock(module: *module_ir.Module, block_id: ids.BlockId) void {
+    const block = module.blocks.get(block_id) orelse return;
+    for (block.parameters.items) |parameter_id|
+        _ = module.values.remove(parameter_id);
+    for (block.instructions.items) |instruction_id| {
+        const inst = module.instructions.get(instruction_id) orelse continue;
+        if (inst.result) |result_id|
+            _ = module.values.remove(result_id);
+        _ = module.instructions.remove(instruction_id);
+    }
+    _ = module.blocks.remove(block_id);
+}
+
+fn mappedBlock(block_map: []const ?ids.BlockId, block_id: ids.BlockId) ?ids.BlockId {
+    if (block_id.index() >= block_map.len)
+        return null;
+    return block_map[block_id.index()];
+}
+
+fn mappedInstruction(instruction_map: []const ?ids.InstructionId, instruction_id: ids.InstructionId) ?ids.InstructionId {
+    if (instruction_id.index() >= instruction_map.len)
+        return null;
+    return instruction_map[instruction_id.index()];
+}
+
+fn removeNonEntryFunctions(module: *module_ir.Module, entry_point: ids.FunctionId) bool {
+    var changed = false;
+    for (module.functions.entries.items, 0..) |entry, function_index| {
+        const function = entry orelse continue;
+        const function_id = ids.FunctionId.fromIndex(function_index);
+        if (function_id == entry_point)
+            continue;
+
+        for (function.parameters.items) |parameter_id|
+            _ = module.values.remove(parameter_id);
+
+        for (function.blocks.items) |block_id| {
+            const block = module.blocks.get(block_id) orelse continue;
+            for (block.parameters.items) |parameter_id|
+                _ = module.values.remove(parameter_id);
+            for (block.instructions.items) |instruction_id| {
+                const inst = module.instructions.get(instruction_id) orelse continue;
+                if (inst.result) |result_id|
+                    _ = module.values.remove(result_id);
+                _ = module.instructions.remove(instruction_id);
+            }
+            _ = module.blocks.remove(block_id);
+        }
+
+        _ = module.functions.remove(function_id);
+        changed = true;
+    }
+    return changed;
+}
+
+fn runForTest(module: *module_ir.Module) !bool {
+    var manager = transformer_manager.Manager.init(std.testing.allocator);
+    defer manager.deinit();
+    try manager.add(transformer);
+
+    var context: transformer_manager.Context = .{ .allocator = std.testing.allocator };
+    return manager.run(module, &context);
+}
+
+fn liveFunctionCount(module: *const module_ir.Module) usize {
+    var count: usize = 0;
+    for (module.functions.entries.items) |entry| {
+        if (entry != null)
+            count += 1;
+    }
+    return count;
+}
+
+test "Inline All Functions: nested calls and multiple returns" {
+    const parser = @import("../parser/parser.zig");
+    const validator = @import("../validator/validator.zig");
+
+    var module = try parser.parseString(std.testing.allocator,
+        \\shader vertex @main
+        \\{
+        \\    %condition: constant bool = true
+        \\    %one: constant u32 = bits(0x1)
+        \\    %two: constant u32 = bits(0x2)
+        \\    fn @main() -> void
+        \\    {
+        \\        .entry():
+        \\            %result: u32 = call @outer(%one)
+        \\            %sum: u32 = integer_add %result, %two
+        \\            return
+        \\    }
+        \\    fn @outer(%outer_value: u32) -> u32
+        \\    {
+        \\        .entry():
+        \\            %nested: u32 = call @identity(%outer_value)
+        \\            return %nested
+        \\    }
+        \\    fn @identity(%identity_value: u32) -> u32
+        \\    {
+        \\        .entry():
+        \\            conditional_branch %condition, .left(), .right()
+        \\        .left():
+        \\            return %identity_value
+        \\        .right():
+        \\            return %two
+        \\    }
+        \\}
+    );
+    defer module.deinit();
+
+    try std.testing.expect(try runForTest(&module));
+    try validator.validate(&module);
+    try std.testing.expect(module.properties.no_function_calls);
+    try std.testing.expectEqual(@as(usize, 1), liveFunctionCount(&module));
+
+    for (module.instructions.entries.items) |entry| {
+        const inst = entry orelse continue;
+        try std.testing.expect(inst.operation != .call);
+    }
+}
+
+test "Inline All Functions: reject reachable recursion" {
+    const parser = @import("../parser/parser.zig");
+
+    var module = try parser.parseString(std.testing.allocator,
+        \\shader vertex @main
+        \\{
+        \\    %one: constant u32 = bits(0x1)
+        \\    fn @main() -> void
+        \\    {
+        \\        .entry():
+        \\            %result: u32 = call @recurse(%one)
+        \\            return
+        \\    }
+        \\    fn @recurse(%value: u32) -> u32
+        \\    {
+        \\        .entry():
+        \\            %nested: u32 = call @recurse(%value)
+        \\            return %nested
+        \\    }
+        \\}
+    );
+    defer module.deinit();
+
+    try std.testing.expectError(Error.RecursiveCall, runForTest(&module));
+    try std.testing.expectEqual(@as(usize, 2), liveFunctionCount(&module));
+}
+
+test "Inline All Functions: remove calls in unreachable blocks" {
+    const parser = @import("../parser/parser.zig");
+    const validator = @import("../validator/validator.zig");
+
+    var module = try parser.parseString(std.testing.allocator,
+        \\shader vertex @main
+        \\{
+        \\    %one: constant u32 = bits(0x1)
+        \\    fn @main() -> void
+        \\    {
+        \\        .entry():
+        \\            return
+        \\        .dead():
+        \\            %local: u32 = integer_add %one, %one
+        \\            %result: u32 = call @identity(%local)
+        \\            return
+        \\    }
+        \\    fn @identity(%identity_value: u32) -> u32
+        \\    {
+        \\        .entry():
+        \\            return %identity_value
+        \\    }
+        \\}
+    );
+    defer module.deinit();
+
+    try std.testing.expect(try runForTest(&module));
+    try validator.validate(&module);
+    try std.testing.expectEqual(@as(usize, 1), liveFunctionCount(&module));
+    const entry_function = module.functions.get(module.entry_point.?).?;
+    try std.testing.expectEqual(@as(usize, 1), entry_function.blocks.items.len);
+}
+
+test "Inline All Functions: remove unreachable recursive helpers" {
+    const parser = @import("../parser/parser.zig");
+    const validator = @import("../validator/validator.zig");
+
+    var module = try parser.parseString(std.testing.allocator,
+        \\shader vertex @main
+        \\{
+        \\    fn @main() -> void
+        \\    {
+        \\        .entry():
+        \\            return
+        \\    }
+        \\    fn @dead() -> void
+        \\    {
+        \\        .entry():
+        \\            call @dead()
+        \\            return
+        \\    }
+        \\}
+    );
+    defer module.deinit();
+
+    try std.testing.expect(try runForTest(&module));
+    try validator.validate(&module);
+    try std.testing.expectEqual(@as(usize, 1), liveFunctionCount(&module));
+}

src/compiler/ir/type.zig

@@ -24,6 +24,10 @@ pub const ArrayType = struct {
     length: u32,
 };
 
+pub const RuntimeArrayType = struct {
+    element_type: TypeId,
+};
+
 pub const StructureType = struct {
     members: []const TypeId,
 };
@@ -68,6 +72,7 @@ pub const Type = union(enum) {
     structure: StructureType,
     pointer: PointerType,
     resource_handle: ResourceHandleType,
+    runtime_array: RuntimeArrayType,
 
     pub fn eql(a: Type, b: Type) bool {
         return switch (a) {
@@ -101,6 +106,10 @@ pub const Type = union(enum) {
                 .resource_handle => |other| std.meta.eql(value, other),
                 else => false,
             },
+            .runtime_array => |value| switch (b) {
+                .runtime_array => |other| std.meta.eql(value, other),
+                else => false,
+            },
         };
     }
 };

src/compiler/ir/validator/dominance.zig

@@ -19,8 +19,8 @@ const DominanceUseContext = struct {
 
 pub fn validate(module: *const module_ir.Module, function_id: ids.FunctionId) Error!void {
     var analysis = cfg.init(module.backingAllocator(), module, function_id) catch |err| switch (err) {
-        error.OutOfMemory => return error.OutOfMemory,
+        std.mem.Allocator.Error.OutOfMemory => return Error.OutOfMemory,
-        else => return error.InvalidBlock,
+        else => return Error.InvalidBlock,
     };
     defer analysis.deinit();
 
@@ -41,7 +41,7 @@ pub fn validate(module: *const module_ir.Module, function_id: ids.FunctionId) Er
             instruction.operation.visitValueUses(&context, checkDominanceUse);
 
             if (!context.valid)
-                return error.DefinitionDoesNotDominateUse;
+                return Error.DefinitionDoesNotDominateUse;
         }
 
         var context: DominanceUseContext = .{
@@ -55,7 +55,7 @@ pub fn validate(module: *const module_ir.Module, function_id: ids.FunctionId) Er
         module_ir.visitTerminatorValueUses(block.terminator.?, &context, checkDominanceUse);
 
         if (!context.valid)
-            return error.DefinitionDoesNotDominateUse;
+            return Error.DefinitionDoesNotDominateUse;
     }
 }
 

src/compiler/root.zig

@@ -3,824 +3,15 @@
 //! This module exposes the project-specific intermediate representation in
 //! `ir` and the SPIR-V frontend in `spirv`.
 //!
-//! Together they form
+//! Together they form the first stage of the compiler pipeline: SPIR-V binary
-//! the first stage of the compiler pipeline: SPIR-V binary modules are decoded,
+//! modules are decoded, translated into a smaller and easier-to-transform IR,
-//! translated into a smaller and easier-to-transform IR, validated, and then made
+//! validated, and then made available to later optimization or code-generation
-//! available to later optimization or code-generation passes.
+//! transformers.
-
-const std = @import("std");
 
 pub const ir = @import("ir/ir.zig");
 pub const spirv = @import("spirv/root.zig");
 
-const VisitorStatistics = struct {
+test {
-    functions: usize = 0,
+    _ = ir;
-    blocks: usize = 0,
+    _ = spirv;
-};
-
-test "IR builder generation" {
-    // shader vertex @main
-    // {
-    //     @color: vec4[f32] = input[location(0), component(0), index(0)]
-    //     @out_color: vec4[f32] = output[location(0), component(0), index(0)]
-    //     %0: constant bool = true
-    //     %1: constant f32 = bits(0x3f800000)
-    //
-    //     fn @main() -> void
-    //     {
-    //         .entry():
-    //             %3: vec4[f32] = load_interface @color
-    //             conditional_branch %0, .pass(), .merge(%3)
-    //
-    //         .pass():
-    //             %4: vec4[f32] = composite_construct %1, %1, %1, %1
-    //             branch .merge(%4)
-    //
-    //         .merge(%2: vec4[f32]):
-    //             store_interface @out_color, %2
-    //             return
-    //     }
-    // }
-
-    var module = ir.module.Module.init(std.testing.allocator, .vertex);
-    defer module.deinit();
-    var builder = ir.Builder.init(&module);
-
-    const void_type = try builder.internType(.void);
-    const bool_type = try builder.internType(.boolean);
-    const f32_type = try builder.internType(.{ .floating = .{ .bits = 32 } });
-    const duplicate_f32 = try builder.internType(.{ .floating = .{ .bits = 32 } });
-    try std.testing.expectEqual(f32_type, duplicate_f32);
-    const vec4_type = try builder.internType(.{ .vector = .{ .element_type = f32_type, .length = 4 } });
-
-    const true_value = try builder.internConstant(bool_type, .{ .boolean = true });
-    const one = try builder.internConstant(f32_type, .{ .float_bits = @as(u32, @bitCast(@as(f32, 1.0))) });
-
-    const input = try builder.addInterfaceVariable(vec4_type, .input, .{ .location = .{ .location = 0 } }, "color");
-    const output = try builder.addInterfaceVariable(vec4_type, .output, .{ .location = .{ .location = 0 } }, "out_color");
-    const main = try builder.addFunction(void_type, "main");
-    builder.setEntryPoint(main);
-    const entry = try builder.addBlock(main, "entry");
-    const pass = try builder.addBlock(main, "pass");
-    const merge = try builder.addBlock(main, "merge");
-    const merged = try builder.addBlockParameter(merge, vec4_type, "merged");
-
-    const loaded = (try builder.appendInstruction(entry, vec4_type, .{
-        .load_interface = .{ .variable = input },
-    }, "loaded")).?;
-    try builder.setTerminator(entry, .{ .conditional_branch = .{
-        .condition = true_value,
-        .true_edge = try builder.edge(pass, &.{}),
-        .false_edge = try builder.edge(merge, &.{loaded}),
-    } });
-
-    const splat = (try builder.appendInstruction(pass, vec4_type, .{
-        .composite_construct = .{ .elements = &.{ one, one, one, one } },
-    }, "white")).?;
-    try builder.setTerminator(pass, .{ .branch = try builder.edge(merge, &.{splat}) });
-    _ = try builder.appendInstruction(merge, null, .{
-        .store_interface = .{ .variable = output, .value = merged },
-    }, null);
-    try builder.setTerminator(merge, .return_void);
-
-    try ir.validator.validate(&module);
-
-    var control_flow = try ir.cfg.init(std.testing.allocator, &module, main);
-    defer control_flow.deinit();
-    try std.testing.expectEqual(@as(usize, 2), control_flow.predecessors(merge).?.len);
-    try std.testing.expect(control_flow.dominates(entry, merge));
-    try std.testing.expect(!control_flow.dominates(pass, merge));
-
-    const text = try ir.printer.allocPrint(std.testing.allocator, &module);
-
-    defer std.testing.allocator.free(text);
-    try std.testing.expect(std.mem.indexOf(u8, text, "shader vertex @main") != null);
-    try std.testing.expect(std.mem.indexOf(u8, text, "@color: vec4[f32] = input[location(0), component(0), index(0)]") != null);
-    try std.testing.expect(std.mem.indexOf(u8, text, "@out_color: vec4[f32] = output[location(0), component(0), index(0)]") != null);
-    try std.testing.expect(std.mem.indexOf(u8, text, "conditional_branch %0, .pass(), .merge(%loaded)") != null);
-    try std.testing.expect(std.mem.indexOf(u8, text, ".merge(%merged: vec4[f32])") != null);
-    try std.testing.expect(std.mem.indexOf(u8, text, "store_interface @out_color, %merged") != null);
-
-    var parsed = try ir.parser.parseString(std.testing.allocator, text);
-    defer parsed.deinit();
-    const round_trip = try ir.printer.allocPrint(std.testing.allocator, &parsed);
-    defer std.testing.allocator.free(round_trip);
-    try std.testing.expectEqualStrings(text, round_trip);
-
-    const io = std.Options.debug_io;
-    const path = ".zig-cache/ir-parser-round-trip.ir";
-    const file = try std.Io.Dir.cwd().createFile(io, path, .{ .truncate = true });
-    {
-        defer file.close(io);
-        var file_buffer: [4096]u8 = @splat(0);
-        var file_writer = file.writer(io, &file_buffer);
-        try file_writer.interface.writeAll(text);
-        try file_writer.interface.flush();
-    }
-    defer std.Io.Dir.cwd().deleteFile(io, path) catch @panic("Caught an error while handling an error");
-
-    var parsed_file = try ir.parser.parseFile(std.testing.allocator, io, path);
-    defer parsed_file.deinit();
-    const file_round_trip = try ir.printer.allocPrint(std.testing.allocator, &parsed_file);
-    defer std.testing.allocator.free(file_round_trip);
-    try std.testing.expectEqualStrings(text, file_round_trip);
-}
-
-test "IR parse interface" {
-    const source =
-        \\ shader vertex @main
-        \\ {
-        \\     @in_color: vec4[f32] = input[location(0), component(0), index(0)]
-        \\     @out_color: vec4[f32] = output[location(0), component(0), index(0)]
-        \\     @position: vec4[f32] = output[builtin(position)]
-        \\
-        \\     fn @main() -> void
-        \\     {
-        \\         .entry():
-        \\             return
-        \\     }
-        \\ }
-    ;
-
-    var module = try ir.parser.parseString(std.testing.allocator, source);
-    defer module.deinit();
-    const printed = try ir.printer.allocPrint(std.testing.allocator, &module);
-    defer std.testing.allocator.free(printed);
-
-    try std.testing.expect(std.mem.indexOf(u8, printed, "@in_color: vec4[f32] = input[location(0), component(0), index(0)]") != null);
-    try std.testing.expect(std.mem.indexOf(u8, printed, "@out_color: vec4[f32] = output[location(0), component(0), index(0)]") != null);
-    try std.testing.expect(std.mem.indexOf(u8, printed, "@position: vec4[f32] = output[builtin(position)]") != null);
-}
-
-test "IR parse types, operations, calls, terminators" {
-    const source =
-        \\ shader fragment @main
-        \\ {
-        \\     %0: constant bool = true
-        \\     %1: constant u32 = bits(0x1)
-        \\     %2: constant u32 = bits(0x2)
-        \\     %3: constant f32 = bits(0x3f800000)
-        \\     %4: constant array[u32, 2] = [#1, #2]
-        \\     %5: constant struct[u32, u32] = [#1, #2]
-        \\     %6: constant ptr[private, u32] = null
-        \\     %7: constant resourceHandle[sampler] = null
-        \\
-        \\     fn @main() -> void
-        \\     {
-        \\         .entry():
-        \\             %9: u32 = bitwise_not %1
-        \\             %10: u32 = integer_add %9, %2
-        \\             %11: bool = cmp_equal %1, %2
-        \\             %12: u32 = select %11, %1, %2
-        \\             %13: u32 = bitcast %12
-        \\             %14: vec2[u32] = composite_construct %1, %2
-        \\             %15: u32 = composite_extract %14[0]
-        \\             %16: f32 = negate %3
-        \\             %17: f32 = float_add %3, %16
-        \\             %18: u32 = call @helper(%15)
-        \\             return
-        \\     }
-        \\
-        \\     fn @helper(%8: u32) -> u32
-        \\     {
-        \\         .entry():
-        \\             return %8
-        \\     }
-        \\
-        \\     fn @discarder() -> void
-        \\     {
-        \\         .entry():
-        \\             discard
-        \\     }
-        \\
-        \\     fn @dead() -> void
-        \\     {
-        \\         .entry():
-        \\             unreachable
-        \\     }
-        \\ }
-    ;
-
-    var module = try ir.parser.parseString(std.testing.allocator, source);
-    defer module.deinit();
-    const printed = try ir.printer.allocPrint(std.testing.allocator, &module);
-    defer std.testing.allocator.free(printed);
-
-    var reparsed = try ir.parser.parseString(std.testing.allocator, printed);
-    defer reparsed.deinit();
-    const printed_again = try ir.printer.allocPrint(std.testing.allocator, &reparsed);
-    defer std.testing.allocator.free(printed_again);
-    try std.testing.expectEqualStrings(printed, printed_again);
-    try std.testing.expect(std.mem.indexOf(u8, printed, "cmp_equal %1, %2") != null);
-    try std.testing.expect(std.mem.indexOf(u8, printed, "cmp.") == null);
-}
-
-test "IR parse named value IDs" {
-    const source =
-        \\ shader compute @main
-        \\ {
-        \\     %one_value: constant u32 = bits(0x1)
-        \\
-        \\     fn @main() -> void
-        \\     {
-        \\         .entry():
-        \\             %sum_value: u32 = integer_add %one_value, %one_value
-        \\             branch .merge(%sum_value)
-        \\
-        \\         .merge(%merged_value: u32):
-        \\             %product_value: u32 = integer_multiply %merged_value, %one_value
-        \\             return
-        \\     }
-        \\ }
-    ;
-
-    var module = try ir.parser.parseString(std.testing.allocator, source);
-    defer module.deinit();
-    const printed = try ir.printer.allocPrint(std.testing.allocator, &module);
-    defer std.testing.allocator.free(printed);
-
-    try std.testing.expect(std.mem.indexOf(u8, printed, "%one_value: constant u32 = bits(0x1)") != null);
-    try std.testing.expect(std.mem.indexOf(u8, printed, "%sum_value: u32 = integer_add %one_value, %one_value") != null);
-    try std.testing.expect(std.mem.indexOf(u8, printed, "branch .merge(%sum_value)") != null);
-    try std.testing.expect(std.mem.indexOf(u8, printed, ".merge(%merged_value: u32)") != null);
-    try std.testing.expect(std.mem.indexOf(u8, printed, "%product_value: u32 = integer_multiply %merged_value, %one_value") != null);
-
-    var reparsed = try ir.parser.parseString(std.testing.allocator, printed);
-    defer reparsed.deinit();
-    const printed_again = try ir.printer.allocPrint(std.testing.allocator, &reparsed);
-    defer std.testing.allocator.free(printed_again);
-    try std.testing.expectEqualStrings(printed, printed_again);
-}
-
-test "IR parse numeric constants" {
-    const source =
-        \\ shader compute @main
-        \\ {
-        \\     %0: constant u8 = 255
-        \\     %1: constant i8 = -1
-        \\     %2: constant f16 = 1.5
-        \\     %3: constant f32 = -0.0
-        \\     %4: constant f64 = 2.5e0
-        \\
-        \\     fn @main() -> void
-        \\     {
-        \\         .entry():
-        \\             return
-        \\     }
-        \\ }
-    ;
-
-    var module = try ir.parser.parseString(std.testing.allocator, source);
-    defer module.deinit();
-    const printed = try ir.printer.allocPrint(std.testing.allocator, &module);
-    defer std.testing.allocator.free(printed);
-
-    try std.testing.expect(std.mem.indexOf(u8, printed, "%0: constant u8 = bits(0xff)") != null);
-    try std.testing.expect(std.mem.indexOf(u8, printed, "%1: constant i8 = bits(0xff)") != null);
-    try std.testing.expect(std.mem.indexOf(u8, printed, "%2: constant f16 = bits(0x3e00)") != null);
-    try std.testing.expect(std.mem.indexOf(u8, printed, "%3: constant f32 = bits(0x80000000)") != null);
-    try std.testing.expect(std.mem.indexOf(u8, printed, "%4: constant f64 = bits(0x4004000000000000)") != null);
-
-    const out_of_range =
-        \\ shader compute @main
-        \\ {
-        \\     %0: constant u8 = 256
-        \\     fn @main() -> void
-        \\     {
-        \\         .entry():
-        \\             return
-        \\     }
-        \\ }
-    ;
-    try std.testing.expectError(error.InvalidNumber, ir.parser.parseString(std.testing.allocator, out_of_range));
-}
-
-test "IR parser error: unknown value" {
-    const source =
-        \\ shader compute @main
-        \\ {
-        \\     fn @main() -> void
-        \\     {
-        \\         .entry():
-        \\             return %99
-        \\     }
-        \\ }
-    ;
-    try std.testing.expectError(error.UnknownValue, ir.parser.parseString(std.testing.allocator, source));
-}
-
-test "Validator error: wrong block argument count" {
-    // shader compute @main
-    // {
-    //     fn @main() -> void
-    //     {
-    //         .entry():
-    //             branch .merge()
-    //
-    //         .merge(%0: u32):
-    //             return
-    //     }
-    // }
-
-    var module = ir.module.Module.init(std.testing.allocator, .compute);
-    defer module.deinit();
-    var builder = ir.Builder.init(&module);
-
-    const void_type = try builder.internType(.void);
-    const u32_type = try builder.internType(.{ .integer = .{ .bits = 32, .signedness = .unsigned } });
-    const main = try builder.addFunction(void_type, "main");
-    builder.setEntryPoint(main);
-    const entry = try builder.addBlock(main, "entry");
-    const merge = try builder.addBlock(main, "merge");
-    _ = try builder.addBlockParameter(merge, u32_type, null);
-    try builder.setTerminator(entry, .{ .branch = try builder.edge(merge, &.{}) });
-    try builder.setTerminator(merge, .return_void);
-
-    try std.testing.expectError(error.WrongBranchArgumentCount, ir.validator.validate(&module));
-}
-
-test "Central store IDs disposal" {
-    var module = ir.module.Module.init(std.testing.allocator, .fragment);
-    defer module.deinit();
-    const first = try module.internType(.boolean);
-    try std.testing.expect(module.types.remove(first));
-    const second = try module.internType(.boolean);
-    try std.testing.expect(first.index() != second.index());
-    try std.testing.expect(module.types.get(first) == null);
-}
-
-test "Validator error: SSA definition does not dominate its use" {
-    // shader compute @main
-    // {
-    //     %0: constant bool = true
-    //     %1: constant u32 = bits(0x1)
-    //
-    //     fn @main() -> void
-    //     {
-    //         .entry():
-    //             conditional_branch %0, .left(), .right()
-    //
-    //         .left():
-    //             %2: u32 = integer_add %1, %1
-    //             branch .merge()
-    //
-    //         .right():
-    //             branch .merge()
-    //
-    //         .merge():
-    //             %3: u32 = integer_multiply %2, %1
-    //             return
-    //     }
-    // }
-
-    var module = ir.module.Module.init(std.testing.allocator, .compute);
-    defer module.deinit();
-    var builder = ir.Builder.init(&module);
-
-    const void_type = try builder.internType(.void);
-    const bool_type = try builder.internType(.boolean);
-    const u32_type = try builder.internType(.{ .integer = .{ .bits = 32, .signedness = .unsigned } });
-    const condition = try builder.internConstant(bool_type, .{ .boolean = true });
-    const one = try builder.internConstant(u32_type, .{ .integer_bits = 1 });
-    const main = try builder.addFunction(void_type, "main");
-    builder.setEntryPoint(main);
-    const entry = try builder.addBlock(main, "entry");
-    const left = try builder.addBlock(main, "left");
-    const right = try builder.addBlock(main, "right");
-    const merge = try builder.addBlock(main, "merge");
-
-    try builder.setTerminator(
-        entry,
-        .{
-            .conditional_branch = .{
-                .condition = condition,
-                .true_edge = try builder.edge(left, &.{}),
-                .false_edge = try builder.edge(right, &.{}),
-            },
-        },
-    );
-
-    const left_value = (try builder.appendInstruction(left, u32_type, .{
-        .binary = .{
-            .opcode = .integer_add,
-            .lhs = one,
-            .rhs = one,
-        },
-    }, null)).?;
-
-    try builder.setTerminator(left, .{ .branch = try builder.edge(merge, &.{}) });
-    try builder.setTerminator(right, .{ .branch = try builder.edge(merge, &.{}) });
-
-    _ = try builder.appendInstruction(merge, u32_type, .{
-        .binary = .{
-            .opcode = .integer_multiply,
-            .lhs = left_value,
-            .rhs = one,
-        },
-    }, null);
-
-    try builder.setTerminator(merge, .return_void);
-
-    try std.testing.expectError(error.DefinitionDoesNotDominateUse, ir.validator.validate(&module));
-}
-
-test "Rewriter replace all ID uses, safely erase dead instruction" {
-    // shader compute @main
-    // {
-    //     %0: constant u32 = bits(0x1)
-    //     %1: constant u32 = bits(0x2)
-    //
-    //     fn @main() -> void
-    //     {
-    //         .entry():
-    //             %2: u32 = integer_add %0, %1
-    //             %3: u32 = integer_multiply %2, %1
-    //             return
-    //     }
-    // }
-
-    var module = ir.module.Module.init(std.testing.allocator, .compute);
-    defer module.deinit();
-
-    var builder = ir.Builder.init(&module);
-
-    const void_type = try builder.internType(.void);
-    const u32_type = try builder.internType(.{ .integer = .{ .bits = 32, .signedness = .unsigned } });
-
-    const one = try builder.internConstant(u32_type, .{ .integer_bits = 1 });
-    const two = try builder.internConstant(u32_type, .{ .integer_bits = 2 });
-
-    const main = try builder.addFunction(void_type, "main");
-    builder.setEntryPoint(main);
-
-    const entry = try builder.addBlock(main, "entry");
-    const sum = (try builder.appendInstruction(entry, u32_type, .{
-        .binary = .{
-            .opcode = .integer_add,
-            .lhs = one,
-            .rhs = two,
-        },
-    }, null)).?;
-    _ = try builder.appendInstruction(entry, u32_type, .{
-        .binary = .{
-            .opcode = .integer_multiply,
-            .lhs = sum,
-            .rhs = two,
-        },
-    }, null);
-    try builder.setTerminator(entry, .return_void);
-
-    try ir.validator.validate(&module);
-
-    const sum_instruction = module.values.get(sum).?.definition.instruction;
-    var rewriter = ir.Rewriter.init(&module);
-
-    try std.testing.expectEqual(@as(usize, 1), try rewriter.replaceAllUses(sum, one));
-    try rewriter.eraseInstruction(sum_instruction);
-
-    try std.testing.expect(module.values.get(sum) == null);
-    try std.testing.expect(module.instructions.get(sum_instruction) == null);
-
-    try ir.validator.validate(&module);
-}
-
-test "Rewriter add block parameter and sync branch calls" {
-    // shader compute @main
-    // {
-    //     %0: constant u32 = bits(0x1)
-    //
-    //     fn @main() -> void
-    //     {
-    //         .entry():
-    //             branch .merge()
-    //
-    //         .merge():
-    //             return
-    //
-    //         .alternate():
-    //             return
-    //     }
-    // }
-
-    var module = ir.module.Module.init(std.testing.allocator, .compute);
-    defer module.deinit();
-
-    var builder = ir.Builder.init(&module);
-
-    const void_type = try builder.internType(.void);
-    const u32_type = try builder.internType(.{ .integer = .{ .bits = 32, .signedness = .unsigned } });
-
-    const one = try builder.internConstant(u32_type, .{ .integer_bits = 1 });
-
-    const main = try builder.addFunction(void_type, "main");
-    builder.setEntryPoint(main);
-
-    const entry = try builder.addBlock(main, "entry");
-    const merge = try builder.addBlock(main, "merge");
-    const alternate = try builder.addBlock(main, "alternate");
-
-    try builder.setTerminator(entry, .{ .branch = try builder.edge(merge, &.{}) });
-    try builder.setTerminator(merge, .return_void);
-    try builder.setTerminator(alternate, .return_void);
-
-    var rewriter = ir.Rewriter.init(&module);
-
-    const parameter = try rewriter.addBlockParameter(merge, u32_type, "incoming", &.{
-        .{
-            .predecessor = entry,
-            .value = one,
-        },
-    });
-    const merge_edge = module.blocks.get(entry).?.terminator.?.branch;
-    try std.testing.expectEqualSlices(ir.id.ValueId, &.{one}, merge_edge.arguments);
-
-    _ = try builder.appendInstruction(merge, u32_type, .{
-        .binary = .{
-            .opcode = .integer_add,
-            .lhs = parameter,
-            .rhs = one,
-        },
-    }, null);
-    try ir.validator.validate(&module);
-
-    try rewriter.removeBlockParameter(merge, 0, one);
-
-    try std.testing.expectEqual(@as(usize, 0), module.blocks.get(merge).?.parameters.items.len);
-    try std.testing.expectEqual(@as(usize, 0), module.blocks.get(entry).?.terminator.?.branch.arguments.len);
-
-    try ir.validator.validate(&module);
-
-    try std.testing.expectEqual(@as(usize, 1), try rewriter.redirectEdges(entry, merge, alternate, &.{}));
-    try std.testing.expectEqual(alternate, module.blocks.get(entry).?.terminator.?.branch.target);
-
-    try ir.validator.validate(&module);
-}
-
-fn establishNoCalls(_: *ir.module.Module, _: *ir.pass_manager.Context) !bool {
-    return false;
-}
-
-fn countVisitedFunction(context: ?*anyopaque, _: ir.id.FunctionId, _: *const ir.module.Function) !void {
-    const statistics: *VisitorStatistics = @ptrCast(@alignCast(context.?));
-    statistics.functions += 1;
-}
-
-fn countVisitedBlock(context: ?*anyopaque, _: ir.id.BlockId, _: *const ir.module.Block) !void {
-    const statistics: *VisitorStatistics = @ptrCast(@alignCast(context.?));
-    statistics.blocks += 1;
-}
-
-test "Pass manager track independent IR properties" {
-    // shader compute @main
-    // {
-    //     fn @main() -> void
-    //     {
-    //         .entry():
-    //             return
-    //     }
-    // }
-
-    var module = ir.module.Module.init(std.testing.allocator, .compute);
-    defer module.deinit();
-
-    var builder = ir.Builder.init(&module);
-
-    const void_type = try builder.internType(.void);
-
-    const main = try builder.addFunction(void_type, "main");
-    builder.setEntryPoint(main);
-
-    const entry = try builder.addBlock(main, "entry");
-    try builder.setTerminator(entry, .return_void);
-
-    module.properties.valid_cfg = true;
-
-    var manager = ir.pass_manager.Manager.init(std.testing.allocator);
-    defer manager.deinit();
-
-    try manager.add(.{
-        .name = "establish-no-calls",
-        .required = .{ .valid_cfg = true },
-        .produced = .{ .no_function_calls = true },
-        .run = establishNoCalls,
-    });
-
-    var context: ir.pass_manager.Context = .{ .allocator = std.testing.allocator };
-
-    try std.testing.expect(!try manager.run(&module, &context));
-    try std.testing.expect(module.properties.no_function_calls);
-
-    var statistics: VisitorStatistics = .{};
-
-    try ir.visitor.walk(&module, .{
-        .context = &statistics,
-        .visitFunction = countVisitedFunction,
-        .visitBlock = countVisitedBlock,
-    });
-
-    try std.testing.expectEqual(@as(usize, 1), statistics.functions);
-    try std.testing.expectEqual(@as(usize, 1), statistics.blocks);
-}
-
-test "SPIR-V parser error: zero-word instruction" {
-    const words = [_]u32{
-        spirv.spec.magic_number,
-        0x0001_0000,
-        0,
-        2,
-        0,
-        instructionWord(.nop, 0),
-    };
-    try std.testing.expectError(error.ZeroWordInstruction, spirv.Parser.init(&words));
-
-    const truncated = [_]u32{
-        spirv.spec.magic_number,
-        0x0001_0000,
-        0,
-        2,
-        0,
-        instructionWord(.i_add, 5),
-        1,
-    };
-    try std.testing.expectError(error.TruncatedInstruction, spirv.Parser.init(&truncated));
-}
-
-test "SPIR-V structured branches and OpPhi to block parameters" {
-    const assembly =
-        \\ OpCapability Shader
-        \\ OpMemoryModel Logical GLSL450
-        \\ OpEntryPoint GLCompute %main "main"
-        \\ OpExecutionMode %main LocalSize 1 1 1
-        \\ OpName %main "main"
-        \\ OpName %entry "entry"
-        \\ OpName %true "true"
-        \\ OpName %one "one"
-        \\ OpName %then "then"
-        \\ OpName %then_value "then_value"
-        \\ OpName %else "else"
-        \\ OpName %else_value "else_value"
-        \\ OpName %merge "merge"
-        \\ OpName %merged "merged"
-        \\ OpName %product "product"
-        \\
-        \\ %void = OpTypeVoid
-        \\ %bool = OpTypeBool
-        \\ %uint = OpTypeInt 32 0
-        \\ %fn_void = OpTypeFunction %void
-        \\ %true = OpConstantTrue %bool
-        \\ %one = OpConstant %uint 1
-        \\
-        \\ %main = OpFunction %void None %fn_void
-        \\     %entry = OpLabel
-        \\     OpSelectionMerge %merge None
-        \\     OpBranchConditional %true %then %else
-        \\     %then = OpLabel
-        \\     %then_value = OpIAdd %uint %one %one
-        \\     OpBranch %merge
-        \\     %else = OpLabel
-        \\     %else_value = OpISub %uint %one %one
-        \\     OpBranch %merge
-        \\     %merge = OpLabel
-        \\     %merged = OpPhi %uint %then_value %then %else_value %else
-        \\     %product = OpIMul %uint %merged %one
-        \\     OpReturn
-        \\ OpFunctionEnd
-    ;
-    const words = try assembleSpirv(std.testing.allocator, assembly);
-    defer std.testing.allocator.free(words);
-
-    var module = try spirv.translator.translate(std.testing.allocator, words, .{ .entry_point = "main" });
-    defer module.deinit();
-
-    try std.testing.expectEqual(ir.module.Stage.compute, module.stage);
-    try std.testing.expectEqual([3]u32{ 1, 1, 1 }, module.execution_modes.workgroup_size.?);
-    try std.testing.expect(module.properties.valid_cfg);
-    try std.testing.expect(module.properties.valid_ssa);
-
-    const function = module.functions.get(module.entry_point.?).?;
-    try std.testing.expectEqual(@as(usize, 4), function.blocks.items.len);
-    const entry = module.blocks.get(function.blocks.items[0]).?;
-    try std.testing.expect(entry.structured_control == .selection);
-    const merge = module.blocks.get(function.blocks.items[3]).?;
-    try std.testing.expectEqual(@as(usize, 1), merge.parameters.items.len);
-    try std.testing.expectEqual(@as(usize, 1), merge.instructions.items.len);
-    const multiply = module.instructions.get(merge.instructions.items[0]).?;
-    try std.testing.expectEqual(ir.instruction.BinaryOpcode.integer_multiply, multiply.operation.binary.opcode);
-
-    const text = try ir.printer.allocPrint(std.testing.allocator, &module);
-    defer std.testing.allocator.free(text);
-
-    try std.testing.expect(std.mem.indexOf(u8, text, "%one: constant u32 = bits(0x1)") != null);
-    try std.testing.expect(std.mem.indexOf(u8, text, "%true: constant bool = true") != null);
-    try std.testing.expect(std.mem.indexOf(u8, text, "conditional_branch %true, .then(), .else()") != null);
-    try std.testing.expect(std.mem.indexOf(u8, text, "%then_value: u32 = integer_add %one, %one") != null);
-    try std.testing.expect(std.mem.indexOf(u8, text, "branch .merge(%then_value)") != null);
-    try std.testing.expect(std.mem.indexOf(u8, text, "%else_value: u32 = integer_subtract %one, %one") != null);
-    try std.testing.expect(std.mem.indexOf(u8, text, ".merge(%merged: u32)") != null);
-    try std.testing.expect(std.mem.indexOf(u8, text, "%product: u32 = integer_multiply %merged, %one") != null);
-    try std.testing.expect(std.mem.indexOf(u8, text, "integerMultiply") == null);
-
-    var parsed = try ir.parser.parseString(std.testing.allocator, text);
-    defer parsed.deinit();
-    const round_trip = try ir.printer.allocPrint(std.testing.allocator, &parsed);
-    defer std.testing.allocator.free(round_trip);
-    try std.testing.expectEqualStrings(text, round_trip);
-}
-
-test "SPIR-V decorated vertex interfaces and load-store operations" {
-    const assembly =
-        \\ OpCapability Shader
-        \\ OpMemoryModel Logical GLSL450
-        \\ OpEntryPoint Vertex %main "main" %in_color %out_color
-        \\ OpName %in_color "in_color"
-        \\ OpName %out_color "out_color"
-        \\ OpDecorate %in_color Location 0
-        \\ OpDecorate %out_color Location 0
-        \\
-        \\ %void = OpTypeVoid
-        \\ %float = OpTypeFloat 32
-        \\ %vec4 = OpTypeVector %float 4
-        \\ %input_vec4 = OpTypePointer Input %vec4
-        \\ %output_vec4 = OpTypePointer Output %vec4
-        \\ %fn_void = OpTypeFunction %void
-        \\ %in_color = OpVariable %input_vec4 Input
-        \\ %out_color = OpVariable %output_vec4 Output
-        \\
-        \\ %main = OpFunction %void None %fn_void
-        \\     %entry = OpLabel
-        \\     %color = OpLoad %vec4 %in_color
-        \\     OpStore %out_color %color
-        \\     OpReturn
-        \\ OpFunctionEnd
-    ;
-    const words = try assembleSpirv(std.testing.allocator, assembly);
-    defer std.testing.allocator.free(words);
-
-    var module = try spirv.translator.translate(std.testing.allocator, words, .{ .entry_point = "main" });
-    defer module.deinit();
-    try std.testing.expectEqual(ir.module.Stage.vertex, module.stage);
-    try std.testing.expectEqual(@as(usize, 2), module.interface_variables.entries.items.len);
-
-    const text = try ir.printer.allocPrint(std.testing.allocator, &module);
-    defer std.testing.allocator.free(text);
-    try std.testing.expect(std.mem.indexOf(u8, text, "load_interface @in_color") != null);
-    try std.testing.expect(std.mem.indexOf(u8, text, "store_interface @out_color") != null);
-}
-
-fn instructionWord(opcode: spirv.spec.Opcode, word_count: u16) u32 {
-    return (@as(u32, word_count) << 16) | @intFromEnum(opcode);
-}
-
-fn assembleSpirv(allocator: std.mem.Allocator, assembly: []const u8) ![]u32 {
-    var io_backend: std.Io.Threaded = .init(allocator, .{});
-    defer io_backend.deinit();
-    const io = io_backend.io();
-
-    var child = try std.process.spawn(io, .{
-        .argv = &.{ "spirv-as", "--target-env", "spv1.0", "-o", "-", "-" },
-        .stdin = .pipe,
-        .stdout = .pipe,
-        .stderr = .pipe,
-    });
-    defer child.kill(io);
-
-    {
-        const stdin = child.stdin.?;
-        var stdin_writer = stdin.writer(io, &.{});
-        try stdin_writer.interface.writeAll(assembly);
-        try stdin_writer.interface.flush();
-        stdin.close(io);
-        child.stdin = null;
-    }
-
-    var stdout_buffer: [4096]u8 = undefined;
-    var stdout_reader = child.stdout.?.reader(io, &stdout_buffer);
-    const binary = try stdout_reader.interface.allocRemaining(allocator, .limited(1024 * 1024));
-    defer allocator.free(binary);
-
-    var stderr_buffer: [4096]u8 = undefined;
-    var stderr_reader = child.stderr.?.reader(io, &stderr_buffer);
-    const stderr = try stderr_reader.interface.allocRemaining(allocator, .limited(64 * 1024));
-    defer allocator.free(stderr);
-
-    const term = try child.wait(io);
-    switch (term) {
-        .exited => |code| if (code != 0) {
-            std.log.err("spirv-as failed:\n{s}", .{stderr});
-            return error.SpirvAssemblyFailed;
-        },
-        else => {
-            std.log.err("spirv-as terminated unexpectedly:\n{s}", .{stderr});
-            return error.SpirvAssemblyFailed;
-        },
-    }
-
-    if (binary.len % @sizeOf(u32) != 0) return error.InvalidSpirvBinaryLength;
-    const words = try allocator.alloc(u32, binary.len / @sizeOf(u32));
-    errdefer allocator.free(words);
-    for (words, 0..) |*word, index| {
-        const offset = index * @sizeOf(u32);
-        word.* = std.mem.readInt(u32, binary[offset..][0..4], .little);
-    }
-    return words;
 }

src/compiler/spirv/Parser.zig

@@ -1,3 +1,4 @@
+const std = @import("std");
 const spirv = @import("spirv.zig");
 
 const Self = @This();
@@ -153,3 +154,135 @@ pub fn copyLiteralString(allocator: anytype, words: []const u32) ![]u8 {
     }
     return result;
 }
+
+test "SPIR-V: parser validates module headers" {
+    const short = [_]u32{ spirv.magic_number, 0x0001_0000, 0, 1 };
+    try std.testing.expectError(error.HeaderTooShort, Self.init(&short));
+
+    var invalid_magic = validHeader(0x0001_0000);
+    invalid_magic[0] = 0x1234_5678;
+    try std.testing.expectError(error.InvalidMagic, Self.init(&invalid_magic));
+
+    var byte_swapped = validHeader(0x0001_0000);
+    byte_swapped[0] = spirv.byte_swapped_magic_number;
+    try std.testing.expectError(error.ByteSwappedModule, Self.init(&byte_swapped));
+
+    var invalid_major = validHeader(0x0002_0000);
+    try std.testing.expectError(error.InvalidVersion, Self.init(&invalid_major));
+
+    var invalid_minor = validHeader(0x0001_0700);
+    try std.testing.expectError(error.InvalidVersion, Self.init(&invalid_minor));
+
+    var invalid_reserved_bits = validHeader(0x0101_0001);
+    try std.testing.expectError(error.InvalidVersion, Self.init(&invalid_reserved_bits));
+
+    var zero_bound = validHeader(0x0001_0000);
+    zero_bound[3] = 0;
+    try std.testing.expectError(error.InvalidIdBound, Self.init(&zero_bound));
+
+    var nonzero_schema = validHeader(0x0001_0000);
+    nonzero_schema[4] = 1;
+    try std.testing.expectError(error.InvalidSchema, Self.init(&nonzero_schema));
+
+    var version_1_6 = validHeader(0x0001_0600);
+    version_1_6[2] = 0xfeed_beef;
+    version_1_6[3] = 42;
+    const parser = try Self.init(&version_1_6);
+    try std.testing.expectEqual(@as(u8, 1), parser.header.major());
+    try std.testing.expectEqual(@as(u8, 6), parser.header.minor());
+    try std.testing.expectEqual(@as(u32, 0xfeed_beef), parser.header.generator);
+    try std.testing.expectEqual(@as(u32, 42), parser.header.bound);
+
+    var instruction_iterator = parser.iterator();
+    try std.testing.expectEqual(@as(?Instruction, null), try instruction_iterator.next());
+}
+
+test "SPIR-V: parser iterates instructions and operands" {
+    const words = [_]u32{
+        spirv.magic_number,
+        0x0001_0000,
+        0,
+        8,
+        0,
+        instructionWord(.nop, 1),
+        instructionWord(.i_add, 5),
+        1,
+        2,
+        3,
+        4,
+    };
+    const parser = try Self.init(&words);
+    var instruction_iterator = parser.iterator();
+
+    const nop = (try instruction_iterator.next()).?;
+    try std.testing.expectEqual(spirv.Opcode.nop, nop.opcode);
+    try std.testing.expectEqual(@as(usize, spirv.header_word_count), nop.word_offset);
+    try std.testing.expectEqual(@as(usize, 0), nop.operands.len);
+    try std.testing.expectEqual(@as(?u32, null), nop.operand(0));
+
+    const add = (try instruction_iterator.next()).?;
+    try std.testing.expectEqual(spirv.Opcode.i_add, add.opcode);
+    try std.testing.expectEqual(@as(usize, spirv.header_word_count + 1), add.word_offset);
+    try std.testing.expectEqualSlices(u32, &.{ 1, 2, 3, 4 }, add.operands);
+    try std.testing.expectEqual(@as(?u32, 1), add.operand(0));
+    try std.testing.expectEqual(@as(?u32, 4), add.operand(3));
+    try std.testing.expectEqual(@as(?u32, null), add.operand(4));
+    try std.testing.expectEqual(@as(?Instruction, null), try instruction_iterator.next());
+}
+
+test "SPIR-V: parser literal string helpers" {
+    const empty = [_]u32{0};
+    try std.testing.expectEqual(@as(usize, 1), try literalStringWordCount(&empty));
+    try std.testing.expect(try literalStringEquals(&empty, ""));
+
+    const abc = [_]u32{0x0063_6261};
+    try std.testing.expectEqual(@as(usize, 1), try literalStringWordCount(&abc));
+    try std.testing.expect(try literalStringEquals(&abc, "abc"));
+    try std.testing.expect(!try literalStringEquals(&abc, "ab"));
+    try std.testing.expect(!try literalStringEquals(&abc, "abcd"));
+
+    const main = [_]u32{ 0x6e69_616d, 0 };
+    try std.testing.expectEqual(@as(usize, 2), try literalStringWordCount(&main));
+    try std.testing.expect(try literalStringEquals(&main, "main"));
+    try std.testing.expect(!try literalStringEquals(&main, "Main"));
+
+    const copy = try copyLiteralString(std.testing.allocator, &main);
+    defer std.testing.allocator.free(copy);
+    try std.testing.expectEqualStrings("main", copy);
+
+    const unterminated = [_]u32{0x6463_6261};
+    try std.testing.expectError(error.UnterminatedString, literalStringWordCount(&unterminated));
+    try std.testing.expectError(error.UnterminatedString, literalStringEquals(&unterminated, "abcd"));
+    try std.testing.expectError(error.UnterminatedString, copyLiteralString(std.testing.allocator, &unterminated));
+}
+
+test "SPIR-V: parser rejects malformed instruction framing" {
+    const words = [_]u32{
+        spirv.magic_number,
+        0x0001_0000,
+        0,
+        2,
+        0,
+        instructionWord(.nop, 0),
+    };
+    try std.testing.expectError(error.ZeroWordInstruction, Self.init(&words));
+
+    const truncated = [_]u32{
+        spirv.magic_number,
+        0x0001_0000,
+        0,
+        2,
+        0,
+        instructionWord(.i_add, 5),
+        1,
+    };
+    try std.testing.expectError(error.TruncatedInstruction, Self.init(&truncated));
+}
+
+fn validHeader(version: u32) [spirv.header_word_count]u32 {
+    return .{ spirv.magic_number, version, 0, 1, 0 };
+}
+
+fn instructionWord(opcode: spirv.Opcode, word_count: u16) u32 {
+    return (@as(u32, word_count) << 16) | @intFromEnum(opcode);
+}

src/compiler/spirv/SourceModule.zig

@@ -0,0 +1,54 @@
+const std = @import("std");
+const Parser = @import("Parser.zig");
+
+const Self = @This();
+
+words: []u32,
+parsed: Parser,
+
+pub const Error = std.mem.Allocator.Error || Parser.Error;
+
+pub fn init(allocator: std.mem.Allocator, words: []const u32) Error!Self {
+    const owned_words = try allocator.dupe(u32, words);
+    errdefer allocator.free(owned_words);
+
+    return .{
+        .words = owned_words,
+        .parsed = try Parser.init(owned_words),
+    };
+}
+
+pub fn deinit(self: *Self, allocator: std.mem.Allocator) void {
+    allocator.free(self.words);
+    self.* = undefined;
+}
+
+pub fn code(self: *const Self) []const u32 {
+    return self.words;
+}
+
+pub fn parser(self: *const Self) Parser {
+    return self.parsed;
+}
+
+test "SPIR-V: source module owns and validates its words" {
+    var words = [_]u32{
+        0x07230203,
+        0x00010000,
+        0,
+        1,
+        0,
+    };
+
+    var source = try Self.init(std.testing.allocator, &words);
+    defer source.deinit(std.testing.allocator);
+
+    words[0] = 0;
+    try std.testing.expectEqual(@as(u32, 0x07230203), source.code()[0]);
+    try std.testing.expectEqual(@as(u8, 1), source.parser().header.major());
+}
+
+test "SPIR-V: source module rejects malformed input" {
+    const malformed = [_]u32{ 0, 0, 0, 0, 0 };
+    try std.testing.expectError(error.InvalidMagic, Self.init(std.testing.allocator, &malformed));
+}

src/compiler/spirv/root.zig

@@ -3,14 +3,21 @@
 //! This namespace contains the SPIR-V parser and translator used to import shader
 //! modules into the compiler IR.
 //!
-//! `Parser` validates the SPIR-V header and iterates over binary instructions.
+//! `Parser` validates borrowed SPIR-V words, while `SourceModule` owns and
-//! `spec` exposes a minimalistic SPIR-V header translation.
+//! structurally validates words that need to outlive an API call. `spec` exposes a
+//! minimalistic SPIR-V header translation.
 //!
-//! The main entry point is `translator.translate`, which finds the requested entry
+//! Use `translator.instantiate` to lower one entry point from a retained source
-//! point, maps its execution model to an IR shader stage, lowers supported types,
+//! module. `translator.translate` remains as a convenience wrapper for borrowed
-//! constants, interfaces, instructions, and structured control flow, then validates
+//! words.
-//! the generated IR module.
 
 pub const Parser = @import("Parser.zig");
+pub const SourceModule = @import("SourceModule.zig");
 pub const translator = @import("translator.zig");
 pub const spec = @import("spirv.zig");
+
+test {
+    _ = Parser;
+    _ = SourceModule;
+    _ = translator;
+}

src/intel/FlintBuffer.zig

@@ -2,7 +2,6 @@ const std = @import("std");
 const vk = @import("vulkan");
 const base = @import("base");
 
-
 const VkError = base.VkError;
 
 const Self = @This();

src/intel/FlintCommandBuffer.zig

@@ -12,6 +12,7 @@ const MemoryRange = @import("MemoryRange.zig");
 
 const copy = @import("copy_commands.zig");
 const blitter = @import("blitter.zig");
+const gen9_dispatch = @import("compiler/targets/gen9/compute/dispatch.zig");
 
 const Self = @This();
 pub const Interface = base.CommandBuffer;
@@ -19,6 +20,8 @@ pub const Interface = base.CommandBuffer;
 interface: Interface,
 batch: std.ArrayList(u32),
 relocations: std.ArrayList(kmd.Relocation),
+gpu_allocations: std.ArrayList(kmd.Memory),
+engine: ?kmd.Engine,
 bound_compute_pipeline: ?*FlintPipeline,
 bound_compute_descriptor_sets: [base.vulkan_max_descriptor_sets]?*FlintDescriptorSet,
 
@@ -84,6 +87,8 @@ pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const v
         .interface = interface,
         .batch = .empty,
         .relocations = .empty,
+        .gpu_allocations = .empty,
+        .engine = null,
         .bound_compute_pipeline = null,
         .bound_compute_descriptor_sets = @splat(null),
     };
@@ -93,8 +98,10 @@ pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const v
 pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
     const command_allocator = self.interface.host_allocator.allocator();
+    self.releaseGpuAllocations();
     self.batch.deinit(command_allocator);
     self.relocations.deinit(command_allocator);
+    self.gpu_allocations.deinit(command_allocator);
     allocator.destroy(self);
 }
 
@@ -105,7 +112,7 @@ pub fn submitGpuBatch(self: *Self, syncs: []const kmd.SyncDependency) VkError!vo
     // Empty command buffers still need a no-op submission to carry queue synchronization.
     const device: *FlintDevice = @alignCast(@fieldParentPtr("interface", self.interface.owner));
     const allocator = self.interface.host_allocator.allocator();
-    try device.kmd.submitBatch(self.interface.owner.io(), allocator, self.batch.items, self.relocations.items, syncs);
+    try device.kmd.submitBatch(self.interface.owner.io(), allocator, self.engine orelse .blitter, self.batch.items, self.relocations.items, syncs);
 }
 
 pub fn begin(interface: *Interface, info: *const vk.CommandBufferBeginInfo) VkError!void {
@@ -119,22 +126,46 @@ pub fn end(interface: *Interface) VkError!void {
 
 pub fn reset(interface: *Interface, flags: vk.CommandBufferResetFlags) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    self.releaseGpuAllocations();
     if (flags.release_resources_bit) {
         const command_allocator = self.interface.host_allocator.allocator();
         self.batch.clearAndFree(command_allocator);
         self.relocations.clearAndFree(command_allocator);
+        self.gpu_allocations.clearAndFree(command_allocator);
     } else {
         self.batch.clearRetainingCapacity();
         self.relocations.clearRetainingCapacity();
+        self.gpu_allocations.clearRetainingCapacity();
     }
+    self.engine = null;
     self.bound_compute_pipeline = null;
     self.bound_compute_descriptor_sets = @splat(null);
 }
 
+fn releaseGpuAllocations(self: *Self) void {
+    const device: *FlintDevice = @alignCast(@fieldParentPtr("interface", self.interface.owner));
+    for (self.gpu_allocations.items) |*allocation|
+        allocation.deinit(&device.kmd, self.interface.owner.io());
+    self.gpu_allocations.clearRetainingCapacity();
+}
+
+pub fn requireEngine(self: *Self, engine: kmd.Engine) VkError!void {
+    if (self.engine) |current| {
+        if (current != engine)
+            return VkError.FeatureNotPresent;
+    } else {
+        self.engine = engine;
+    }
+}
+
 pub fn emit(self: *Self, dword: u32) VkError!void {
     self.batch.append(self.interface.host_allocator.allocator(), dword) catch return VkError.OutOfHostMemory;
 }
 
+fn emitSlice(self: *Self, words: []const u32) VkError!void {
+    self.batch.appendSlice(self.interface.host_allocator.allocator(), words) catch return VkError.OutOfHostMemory;
+}
+
 pub fn emitRelocatedAddress(self: *Self, range: MemoryRange, read: bool, write: bool) VkError!void {
     const address_offset = self.batch.items.len * @sizeOf(u32);
     try self.emit(@intCast(range.offset));
@@ -145,6 +176,7 @@ pub fn emitRelocatedAddress(self: *Self, range: MemoryRange, read: bool, write:
         .delta = @intCast(range.offset),
         .read = read,
         .write = write,
+        .domain = if ((self.engine orelse .blitter) == .render) .render else .none,
     }) catch return VkError.OutOfHostMemory;
 }
 
@@ -174,16 +206,24 @@ pub fn beginRenderPass(interface: *Interface, render_pass: *base.RenderPass, fra
 
 pub fn bindDescriptorSets(interface: *Interface, bind_point: vk.PipelineBindPoint, first_set: u32, sets: [base.vulkan_max_descriptor_sets]?*base.DescriptorSet, dynamic_offsets: []const u32) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    if (bind_point != .compute) return;
+    if (bind_point != .compute)
-    if (first_set >= base.vulkan_max_descriptor_sets) return VkError.ValidationFailed;
+        return;
+
+    if (dynamic_offsets.len != 0)
+        return VkError.FeatureNotPresent;
+
+    if (first_set >= base.vulkan_max_descriptor_sets)
+        return VkError.ValidationFailed;
 
     for (sets, 0..) |set, index| {
         const base_set = set orelse break;
         const destination = first_set + index;
-        if (destination >= base.vulkan_max_descriptor_sets) return VkError.ValidationFailed;
+
+        if (destination >= base.vulkan_max_descriptor_sets)
+            return VkError.ValidationFailed;
+
         self.bound_compute_descriptor_sets[destination] = @alignCast(@fieldParentPtr("interface", base_set));
     }
-    _ = dynamic_offsets;
 }
 
 pub fn bindPipeline(interface: *Interface, bind_point: vk.PipelineBindPoint, pipeline: *base.Pipeline) VkError!void {
@@ -289,13 +329,146 @@ pub fn dispatch(interface: *Interface, group_count_x: u32, group_count_y: u32, g
 }
 
 pub fn dispatchBase(interface: *Interface, base_group_x: u32, base_group_y: u32, base_group_z: u32, group_count_x: u32, group_count_y: u32, group_count_z: u32) VkError!void {
-    _ = interface;
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = base_group_x;
+    if (group_count_x == 0 or group_count_y == 0 or group_count_z == 0)
-    _ = base_group_y;
+        return;
-    _ = base_group_z;
+    if (base_group_x != 0 or base_group_y != 0 or base_group_z != 0 or
-    _ = group_count_x;
+        group_count_x != 1 or group_count_y != 1 or group_count_z != 1)
-    _ = group_count_y;
+        return VkError.FeatureNotPresent;
-    _ = group_count_z;
+
+    const pipeline = self.bound_compute_pipeline orelse return VkError.ValidationFailed;
+    const artifact = pipeline.computeArtifact() orelse return VkError.FeatureNotPresent;
+    const kernel = artifact.kernel orelse return VkError.FeatureNotPresent;
+    if (!std.mem.eql(u32, &artifact.program.workgroup_size, &.{ 1, 1, 1 }) or
+        artifact.program.program_data.scratch_size_bytes != 0)
+        return VkError.FeatureNotPresent;
+
+    var ranges: [gen9_dispatch.max_surfaces]?MemoryRange = @splat(null);
+    var sizes: [gen9_dispatch.max_surfaces]u64 = @splat(0);
+    for (artifact.resources.bindings) |resource| {
+        if (resource.set >= base.vulkan_max_descriptor_sets or @as(usize, resource.binding_table_index) >= gen9_dispatch.max_storage_surfaces)
+            return VkError.ValidationFailed;
+
+        const descriptor_set = self.bound_compute_descriptor_sets[resource.set] orelse return VkError.ValidationFailed;
+        const expected_layout = pipeline.interface.layout.set_layouts[resource.set] orelse return VkError.ValidationFailed;
+        if (descriptor_set.interface.layout != expected_layout)
+            return VkError.ValidationFailed;
+
+        const descriptor = try descriptor_set.getBuffer(resource.binding, 0);
+        const buffer = descriptor.buffer orelse return VkError.ValidationFailed;
+        if (!buffer.usage.storage_buffer_bit or buffer.memory == null)
+            return VkError.ValidationFailed;
+
+        const range = try MemoryRange.fromBuffer(buffer, descriptor.offset, descriptor.size);
+        ranges[resource.binding_table_index] = range;
+        sizes[resource.binding_table_index] = range.size;
+    }
+
+    const old_engine = self.engine;
+    try self.requireEngine(.render);
+    const old_batch_len = self.batch.items.len;
+    const old_relocation_len = self.relocations.items.len;
+    const old_allocation_len = self.gpu_allocations.items.len;
+    errdefer {
+        self.engine = old_engine;
+        self.batch.items.len = old_batch_len;
+        self.relocations.items.len = old_relocation_len;
+        while (self.gpu_allocations.items.len > old_allocation_len) {
+            const device: *FlintDevice = @alignCast(@fieldParentPtr("interface", self.interface.owner));
+            self.gpu_allocations.items[self.gpu_allocations.items.len - 1].deinit(&device.kmd, self.interface.owner.io());
+            self.gpu_allocations.items.len -= 1;
+        }
+    }
+
+    const device: *FlintDevice = @alignCast(@fieldParentPtr("interface", self.interface.owner));
+    var state = try device.kmd.allocateMemory(self.interface.owner.io(), gen9_dispatch.page_size);
+    var state_owned = true;
+    errdefer if (state_owned) state.deinit(&device.kmd, self.interface.owner.io());
+
+    const mapped = try state.map(&device.kmd, self.interface.owner.io(), 0, gen9_dispatch.page_size);
+    const state_layout = gen9_dispatch.writeState(mapped, kernel, sizes[0..artifact.resources.bindings.len]) catch |err| switch (err) {
+        error.StateTooLarge,
+        error.UnsupportedBufferSize,
+        error.EmptyBuffer,
+        error.TooManySurfaces,
+        => return VkError.FeatureNotPresent,
+    };
+    state.unmap();
+    try state.flushRange(&device.kmd, self.interface.owner.io(), 0, state_layout.size);
+    const state_handle = try state.handle();
+
+    self.gpu_allocations.append(self.interface.host_allocator.allocator(), state) catch return VkError.OutOfHostMemory;
+    state_owned = false;
+
+    for (0..@as(usize, state_layout.storage_surface_count)) |index| {
+        const range = ranges[index] orelse return VkError.ValidationFailed;
+        if (range.offset > std.math.maxInt(u32))
+            return VkError.FeatureNotPresent;
+        self.relocations.append(self.interface.host_allocator.allocator(), .{
+            .source_handle = state_handle,
+            .target_handle = try range.memory.allocation.handle(),
+            .offset = state_layout.surface_address_offsets[index],
+            .delta = @intCast(range.offset),
+            .read = true,
+            .write = true,
+            .domain = .render,
+        }) catch return VkError.OutOfHostMemory;
+    }
+
+    const size_table_surface: usize = state_layout.storage_surface_count;
+    self.relocations.append(self.interface.host_allocator.allocator(), .{
+        .source_handle = state_handle,
+        .target_handle = state_handle,
+        .offset = state_layout.surface_address_offsets[size_table_surface],
+        .delta = state_layout.size_table_offset,
+        .read = true,
+        .write = false,
+        .domain = .render,
+    }) catch return VkError.OutOfHostMemory;
+
+    try self.emitSlice(&gen9_dispatch.pipeControl(gen9_dispatch.pipe_control.cs_stall |
+        gen9_dispatch.pipe_control.dc_flush |
+        gen9_dispatch.pipe_control.render_target_flush |
+        gen9_dispatch.pipe_control.depth_flush));
+    try self.emitSlice(&gen9_dispatch.pipeControl(gen9_dispatch.pipe_control.cs_stall |
+        gen9_dispatch.pipe_control.texture_invalidate |
+        gen9_dispatch.pipe_control.constant_invalidate |
+        gen9_dispatch.pipe_control.state_invalidate |
+        gen9_dispatch.pipe_control.instruction_invalidate));
+    try self.emitSlice(&gen9_dispatch.ccStatePointers);
+    try self.emitSlice(&gen9_dispatch.pipelineSelectGpgpu);
+    try self.emitSlice(&gen9_dispatch.pipeControl(gen9_dispatch.pipe_control.cs_stall |
+        gen9_dispatch.pipe_control.dc_flush |
+        gen9_dispatch.pipe_control.render_target_flush));
+
+    const sba_start = self.batch.items.len * @sizeOf(u32);
+    try self.emitSlice(&gen9_dispatch.stateBaseAddress());
+    inline for (.{
+        .{ 4, kmd.Domain.render },
+        .{ 6, kmd.Domain.render },
+        .{ 10, kmd.Domain.instruction },
+    }) |base_address| {
+        self.relocations.append(self.interface.host_allocator.allocator(), .{
+            .target_handle = state_handle,
+            .offset = sba_start + base_address[0] * @sizeOf(u32),
+            .delta = gen9_dispatch.base_address_delta,
+            .read = true,
+            .domain = base_address[1],
+        }) catch return VkError.OutOfHostMemory;
+    }
+
+    try self.emitSlice(&gen9_dispatch.pipeControl(gen9_dispatch.pipe_control.cs_stall |
+        gen9_dispatch.pipe_control.texture_invalidate |
+        gen9_dispatch.pipe_control.constant_invalidate |
+        gen9_dispatch.pipe_control.state_invalidate |
+        gen9_dispatch.pipe_control.instruction_invalidate));
+    try self.emitSlice(&gen9_dispatch.pipeControl(gen9_dispatch.pipe_control.cs_stall));
+    try self.emitSlice(&gen9_dispatch.mediaVfeState());
+    try self.emitSlice(&gen9_dispatch.interfaceDescriptorLoad(state_layout.interface_descriptor_offset));
+    try self.emitSlice(&gen9_dispatch.gpgpuWalker(.{ 1, 1, 1 }, 1));
+    try self.emitSlice(&gen9_dispatch.mediaStateFlush);
+    try self.emitSlice(&gen9_dispatch.pipeControl(gen9_dispatch.pipe_control.cs_stall |
+        gen9_dispatch.pipe_control.dc_flush));
 }
 
 pub fn setDeviceMask(interface: *Interface, device_mask: u32) VkError!void {
@@ -349,23 +522,30 @@ pub fn endRenderPass(interface: *Interface) VkError!void {
 pub fn executeCommands(interface: *Interface, commands: *Interface) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
     const secondary: *Self = @alignCast(@fieldParentPtr("interface", commands));
+    if (secondary.gpu_allocations.items.len != 0)
+        return VkError.FeatureNotPresent;
+    if (secondary.engine) |engine|
+        try self.requireEngine(engine);
+
     const allocator = self.interface.host_allocator.allocator();
     const relocation_offset = self.batch.items.len * @sizeOf(u32);
-
     self.batch.appendSlice(allocator, secondary.batch.items) catch return VkError.OutOfHostMemory;
     for (secondary.relocations.items) |relocation| {
         self.relocations.append(allocator, .{
+            .source_handle = relocation.source_handle,
             .target_handle = relocation.target_handle,
-            .offset = relocation.offset + relocation_offset,
+            .offset = relocation.offset + if (relocation.source_handle == null) relocation_offset else 0,
             .delta = relocation.delta,
             .read = relocation.read,
             .write = relocation.write,
+            .domain = relocation.domain,
         }) catch return VkError.OutOfHostMemory;
     }
 }
 
 pub fn fillBuffer(interface: *Interface, buffer: *base.Buffer, offset: vk.DeviceSize, size: vk.DeviceSize, data: u32) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    try self.requireEngine(.blitter);
     const dst_range = try copy.fillRange(buffer, offset, size);
 
     var filled: vk.DeviceSize = 0;

src/intel/FlintInstance.zig

@@ -31,6 +31,8 @@ pub const extensions = [_]vk.ExtensionProperties{
     castExtension(vk.extensions.khr_get_physical_device_properties_2),
     castExtension(vk.extensions.khr_surface),
     castExtension(vk.extensions.khr_wayland_surface),
+    castExtension(vk.extensions.khr_xlib_surface),
+    castExtension(vk.extensions.khr_xcb_surface),
 };
 
 pub fn create(allocator: std.mem.Allocator, infos: *const vk.InstanceCreateInfo) VkError!*Interface {

src/intel/FlintPhysicalDevice.zig

@@ -7,6 +7,7 @@ const lib = @import("lib.zig");
 const pci_ids = @import("pci_ids.zig").map;
 
 const FlintDevice = @import("FlintDevice.zig");
+const compiler_device = @import("compiler/device.zig");
 
 const VkError = base.VkError;
 const SurfaceKHR = base.SurfaceKHR;
@@ -29,6 +30,7 @@ pub const extensions = [_]vk.ExtensionProperties{
 
 interface: Interface,
 kmd_type: lib.KmdType,
+compiler_info: ?compiler_device.DeviceInfo,
 node_path: [base.drm.max_node_name:0]u8,
 
 pub fn create(allocator: std.mem.Allocator, instance: *base.Instance, drm_device: *const base.drm.Device, kmd_type: lib.KmdType) VkError!*Self {
@@ -224,6 +226,7 @@ pub fn create(allocator: std.mem.Allocator, instance: *base.Instance, drm_device
     self.* = .{
         .interface = interface,
         .kmd_type = kmd_type,
+        .compiler_info = compiler_device.DeviceInfo.fromPciDeviceId(interface.props.device_id),
         .node_path = @splat(0),
     };
     const node_path = drm_device.nodePath();

src/intel/FlintPipeline.zig

@@ -1,47 +1,325 @@
 const std = @import("std");
 const vk = @import("vulkan");
 const base = @import("base");
+const shader_ir = @import("shader_ir");
+const compiler = @import("compiler/compiler.zig");
+const FlintPhysicalDevice = @import("FlintPhysicalDevice.zig");
 
 const VkError = base.VkError;
 
 const Self = @This();
 pub const Interface = base.Pipeline;
 
+const PipelineKind = enum {
+    graphics,
+    compute,
+};
+
+pub const ComputeArtifact = compiler.targets.ComputeArtifact;
+
+const CommonStage = struct {
+    stage: shader_ir.ir.module.Stage,
+    module: base.ShaderModule.IrModule,
+    artifact: ?ComputeArtifact,
+
+    fn deinit(self: *CommonStage, allocator: std.mem.Allocator) void {
+        if (self.artifact) |*artifact|
+            artifact.deinit(allocator);
+        self.module.deinit();
+        self.* = undefined;
+    }
+};
+
 interface: Interface,
-host_allocator: base.VulkanAllocator,
+artifact_allocator: base.VulkanAllocator,
+stages: []CommonStage,
 
 pub fn createCompute(device: *base.Device, allocator: std.mem.Allocator, cache: ?*base.PipelineCache, info: *const vk.ComputePipelineCreateInfo) VkError!*Self {
     const self = allocator.create(Self) catch return VkError.OutOfHostMemory;
-    errdefer allocator.destroy(self);
+    var initialized = false;
+    errdefer if (initialized) self.interface.destroy(allocator) else allocator.destroy(self);
 
     var interface = try Interface.initCompute(device, allocator, cache, info);
     interface.vtable = &.{ .destroy = destroy };
+
     self.* = .{
         .interface = interface,
-        .host_allocator = base.VulkanAllocator.from(allocator).clone(),
+        .artifact_allocator = base.VulkanAllocator.from(allocator).clone(),
+        .stages = &.{},
     };
-    errdefer self.interface.layout.unref(allocator);
+    initialized = true;
 
+    self.stages = try compileStages(self.artifact_allocator.allocator(), device.io(), &.{info.stage}, .compute, compilerDeviceInfo(device));
+    if (self.computeArtifact()) |artifact|
+        try validateComputePipelineLayout(self.interface.layout, &artifact.resources);
     return self;
 }
 
 pub fn createGraphics(device: *base.Device, allocator: std.mem.Allocator, cache: ?*base.PipelineCache, info: *const vk.GraphicsPipelineCreateInfo) VkError!*Self {
     const self = allocator.create(Self) catch return VkError.OutOfHostMemory;
-    errdefer allocator.destroy(self);
+    var initialized = false;
+    errdefer if (initialized) self.interface.destroy(allocator) else allocator.destroy(self);
 
     var interface = try Interface.initGraphics(device, allocator, cache, info);
     interface.vtable = &.{ .destroy = destroy };
 
     self.* = .{
         .interface = interface,
-        .host_allocator = base.VulkanAllocator.from(allocator).clone(),
+        .artifact_allocator = base.VulkanAllocator.from(allocator).clone(),
+        .stages = &.{},
     };
-    errdefer self.interface.layout.unref(allocator);
+    initialized = true;
 
+    const stage_infos = if (info.p_stages) |stages|
+        stages[0..info.stage_count]
+    else
+        return VkError.ValidationFailed;
+    self.stages = try compileStages(self.artifact_allocator.allocator(), device.io(), stage_infos, .graphics, compilerDeviceInfo(device));
     return self;
 }
 
+fn compileStages(allocator: std.mem.Allocator, io: std.Io, infos: []const vk.PipelineShaderStageCreateInfo, pipeline_kind: PipelineKind, device_info: ?compiler.device.DeviceInfo) VkError![]CommonStage {
+    if (infos.len == 0)
+        return VkError.ValidationFailed;
+
+    const stages = allocator.alloc(CommonStage, infos.len) catch return VkError.OutOfHostMemory;
+    var initialized: usize = 0;
+    errdefer {
+        for (stages[0..initialized]) |*stage|
+            stage.deinit(allocator);
+        allocator.free(stages);
+    }
+
+    for (infos, stages) |*info, *stage| {
+        stage.* = try compileStage(allocator, io, info, pipeline_kind, device_info);
+        initialized += 1;
+    }
+    return stages;
+}
+
+fn compileStage(allocator: std.mem.Allocator, io: std.Io, info: *const vk.PipelineShaderStageCreateInfo, pipeline_kind: PipelineKind, device_info: ?compiler.device.DeviceInfo) VkError!CommonStage {
+    const specializations = try specializationValues(allocator, info.p_specialization_info);
+    defer if (specializations.len != 0) allocator.free(specializations);
+
+    const expected_stage = commonStage(info.stage) orelse return VkError.ValidationFailed;
+    switch (pipeline_kind) {
+        .compute => if (expected_stage != .compute) return VkError.ValidationFailed,
+        .graphics => if (expected_stage == .compute) return VkError.ValidationFailed,
+    }
+
+    const shader_module = try base.NonDispatchable(base.ShaderModule).fromHandleObject(info.module);
+    var module = shader_module.instantiateIr(allocator, .{
+        .entry_point = std.mem.span(info.p_name),
+        .stage = expected_stage,
+        .specializations = specializations,
+    }) catch |err| {
+        std.log.scoped(.FlintPipeline).err("common shader translation failed: {s}", .{@errorName(err)});
+        return switch (err) {
+            error.OutOfMemory => VkError.OutOfHostMemory,
+            else => VkError.ValidationFailed,
+        };
+    };
+    errdefer module.deinit();
+
+    std.debug.assert(module.stage == expected_stage);
+    if (base.config.flint_dump_common_ir)
+        dumpCommonIr(allocator, io, std.mem.span(info.p_name), &module);
+
+    var artifact = try lowerToFlint(allocator, &module, device_info);
+    errdefer if (artifact) |*value| value.deinit(allocator);
+    if (base.config.flint_dump_ir) {
+        if (artifact) |*value|
+            dumpFlintIr(allocator, io, std.mem.span(info.p_name), &value.program);
+    }
+
+    return .{
+        .stage = expected_stage,
+        .module = module,
+        .artifact = artifact,
+    };
+}
+
+fn dumpCommonIr(allocator: std.mem.Allocator, io: std.Io, entry_point: []const u8, module: *const base.ShaderModule.IrModule) void {
+    const text = shader_ir.ir.printer.allocPrint(allocator, module) catch |err| {
+        std.log.scoped(.FlintPipeline).err("could not print backend-agnostic IR: {s}", .{@errorName(err)});
+        return;
+    };
+    defer allocator.free(text);
+
+    var stdout_buffer: [1024]u8 = undefined;
+    var stdout_file_writer: std.Io.File.Writer = .init(.stdout(), io, &stdout_buffer);
+    const stdout_writer = &stdout_file_writer.interface;
+    stdout_writer.print("\n=== backend-agnostic IR: {s} ===\n{s}\n", .{ entry_point, text }) catch @panic("Debug printing failed");
+    stdout_writer.flush() catch @panic("Debug printing failed");
+}
+
+fn dumpFlintIr(allocator: std.mem.Allocator, io: std.Io, entry_point: []const u8, program: *const compiler.program.Program) void {
+    const text = compiler.printer.allocPrint(allocator, program) catch |err| {
+        std.log.scoped(.FlintPipeline).err("could not print Flint IR: {s}", .{@errorName(err)});
+        return;
+    };
+    defer allocator.free(text);
+
+    var stdout_buffer: [1024]u8 = undefined;
+    var stdout_file_writer: std.Io.File.Writer = .init(.stdout(), io, &stdout_buffer);
+    const stdout_writer = &stdout_file_writer.interface;
+    stdout_writer.print("\n=== Flint IR: {s} ===\n{s}\n", .{ entry_point, text }) catch @panic("Debug printing failed");
+    stdout_writer.flush() catch @panic("Debug printing failed");
+}
+
+fn lowerToFlint(allocator: std.mem.Allocator, module: *base.ShaderModule.IrModule, device_info: ?compiler.device.DeviceInfo) VkError!?ComputeArtifact {
+    const target = device_info orelse return null;
+    return compiler.targets.compileCompute(allocator, module, target, .{}) catch |err| switch (err) {
+        error.OutOfMemory => return VkError.OutOfHostMemory,
+        else => {
+            std.log.scoped(.FlintPipeline).err("compute compilation failed: {s}", .{@errorName(err)});
+            return VkError.ValidationFailed;
+        },
+    };
+}
+
+fn validateComputePipelineLayout(layout: *const base.PipelineLayout, resources: *const compiler.targets.ComputeResourceLayout) VkError!void {
+    for (resources.bindings) |resource| {
+        if (resource.set >= layout.set_count)
+            return VkError.ValidationFailed;
+
+        const set_layout = layout.set_layouts[resource.set] orelse return VkError.ValidationFailed;
+
+        if (resource.binding >= set_layout.bindings.len)
+            return VkError.ValidationFailed;
+
+        const binding = set_layout.bindings[resource.binding];
+
+        if (binding.descriptor_type != .storage_buffer or binding.array_size == 0)
+            return VkError.ValidationFailed;
+    }
+}
+
+fn compilerDeviceInfo(device: *const base.Device) ?compiler.device.DeviceInfo {
+    const physical_device: *const FlintPhysicalDevice = @alignCast(@fieldParentPtr("interface", device.physical_device));
+    return physical_device.compiler_info;
+}
+
+fn specializationValues(allocator: std.mem.Allocator, info: ?*const vk.SpecializationInfo) VkError![]shader_ir.spirv.translator.SpecializationValue {
+    const specialization = info orelse return &.{};
+    if (specialization.map_entry_count == 0)
+        return &.{};
+
+    const entries = specialization.p_map_entries orelse return VkError.ValidationFailed;
+    const data: []const u8 = if (specialization.data_size == 0)
+        &.{}
+    else
+        @as([*]const u8, @ptrCast(@alignCast(specialization.p_data)))[0..specialization.data_size];
+
+    const values = allocator.alloc(shader_ir.spirv.translator.SpecializationValue, specialization.map_entry_count) catch
+        return VkError.OutOfHostMemory;
+    errdefer allocator.free(values);
+
+    for (entries[0..specialization.map_entry_count], values) |entry, *value| {
+        const offset: usize = entry.offset;
+        const end = std.math.add(usize, offset, entry.size) catch return VkError.ValidationFailed;
+        if (end > data.len)
+            return VkError.ValidationFailed;
+        value.* = .{
+            .constant_id = entry.constant_id,
+            .data = data[offset..end],
+        };
+    }
+    return values;
+}
+
+fn commonStage(stage: vk.ShaderStageFlags) ?shader_ir.ir.module.Stage {
+    const bits: u32 = @bitCast(stage);
+    const vertex_bits: u32 = @bitCast(vk.ShaderStageFlags{ .vertex_bit = true });
+    const fragment_bits: u32 = @bitCast(vk.ShaderStageFlags{ .fragment_bit = true });
+    const compute_bits: u32 = @bitCast(vk.ShaderStageFlags{ .compute_bit = true });
+
+    return if (bits == vertex_bits)
+        .vertex
+    else if (bits == fragment_bits)
+        .fragment
+    else if (bits == compute_bits)
+        .compute
+    else
+        null;
+}
+
+fn deinitStages(allocator: std.mem.Allocator, stages: []CommonStage) void {
+    for (stages) |*stage|
+        stage.deinit(allocator);
+    if (stages.len != 0)
+        allocator.free(stages);
+}
+
+pub fn computeArtifact(self: *const Self) ?*const ComputeArtifact {
+    if (self.interface.bind_point != .compute or self.stages.len != 1 or self.stages[0].stage != .compute)
+        return null;
+    return if (self.stages[0].artifact) |*artifact| artifact else null;
+}
+
 pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    deinitStages(self.artifact_allocator.allocator(), self.stages);
     allocator.destroy(self);
 }
+
+test "Flint pipeline: lower common compute IR" {
+    const device_info: compiler.device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var module = shader_ir.ir.module.Module.init(std.testing.allocator, .compute);
+    defer module.deinit();
+    module.execution_modes.workgroup_size = .{ 1, 1, 1 };
+    var builder = shader_ir.ir.Builder.init(&module);
+
+    const void_type = try builder.internType(.void);
+    const u32_type = try builder.internType(.{ .integer = .{ .bits = 32, .signedness = .unsigned } });
+    const vec3_type = try builder.internType(.{ .vector = .{ .element_type = u32_type, .length = 3 } });
+    const global_id = try builder.addInterfaceVariable(vec3_type, .input, .{ .builtin = .global_invocation_id }, "global_id");
+    const storage = try builder.addResource(u32_type, .storage_buffer, 0, 2, "storage");
+    const zero = try builder.internConstant(u32_type, .{ .integer_bits = 0 });
+    const main = try builder.addFunction(void_type, "main");
+    builder.setEntryPoint(main);
+    const entry = try builder.addBlock(main, "entry");
+    const id = (try builder.appendInstruction(entry, vec3_type, .{
+        .load_interface = .{ .variable = global_id },
+    }, "id")).?;
+    const x = (try builder.appendInstruction(entry, u32_type, .{
+        .composite_extract = .{ .composite = id, .indices = &.{0} },
+    }, "x")).?;
+    _ = try builder.appendInstruction(entry, null, .{
+        .store_buffer = .{ .resource = storage, .byte_offset = zero, .value = x },
+    }, null);
+    try builder.setTerminator(entry, .return_void);
+
+    var artifact = (try lowerToFlint(std.testing.allocator, &module, device_info)).?;
+    defer artifact.deinit(std.testing.allocator);
+    const program = &artifact.program;
+
+    try std.testing.expect(program.properties.common_ir_lowered);
+    try std.testing.expect(program.properties.compute_abi_lowered);
+    try std.testing.expectEqual(@as(u16, 1), program.program_data.payload_grf_count);
+    try std.testing.expectEqual(@as(u16, 0), program.payload.header_grf.?.number);
+    try std.testing.expect(program.properties.block_parameters_lowered);
+    try std.testing.expect(program.properties.system_values_lowered);
+    try std.testing.expect(program.properties.resources_lowered);
+    try std.testing.expect(program.properties.messages_lowered);
+    try std.testing.expect(program.properties.message_addresses_lowered);
+    try std.testing.expect(program.properties.message_payloads_lowered);
+    try std.testing.expect(program.properties.registers_allocated);
+    try std.testing.expect(!program.properties.instructions_selected);
+    try std.testing.expectEqual([3]u32{ 1, 1, 1 }, program.workgroup_size);
+    try std.testing.expectEqual(@as(usize, 1), program.storage_buffers.entries.items.len);
+    try std.testing.expectEqual(@as(usize, 1), artifact.resources.bindings.len);
+    try std.testing.expectEqual(@as(u8, 0), artifact.resources.bindings[0].binding_table_index);
+    try compiler.targets.validate(program);
+
+    const text = try compiler.printer.allocPrint(std.testing.allocator, program);
+    defer std.testing.allocator.free(text);
+    try std.testing.expect(std.mem.indexOf(u8, text, "load_global_invocation_id") == null);
+    try std.testing.expect(std.mem.indexOf(u8, text, "mov r1:u32, 0:u32") != null);
+    try std.testing.expect(std.mem.indexOf(u8, text, "surface_message write bti(0)") != null);
+}

src/intel/FlintQueue.zig

@@ -97,6 +97,7 @@ pub fn submit(interface: *Interface, infos: []Interface.SubmitInfo, fence: ?*bas
                 try device.kmd.submitBatch(
                     interface.owner.io(),
                     allocator,
+                    .blitter,
                     &.{},
                     &.{},
                     syncs.items,
@@ -129,6 +130,7 @@ pub fn submit(interface: *Interface, infos: []Interface.SubmitInfo, fence: ?*bas
         try device.kmd.submitBatch(
             interface.owner.io(),
             allocator,
+            .blitter,
             &.{},
             &.{},
             syncs[0..sync_count],

src/intel/FlintShaderModule.zig

@@ -8,7 +8,6 @@ const Self = @This();
 pub const Interface = base.ShaderModule;
 
 interface: Interface,
-code: []u32,
 ref_count: std.atomic.Value(usize),
 
 pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const vk.ShaderModuleCreateInfo) VkError!*Self {
@@ -16,14 +15,11 @@ pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const v
     errdefer allocator.destroy(self);
 
     var interface = try Interface.init(device, allocator, info);
+    errdefer interface.deinit();
     interface.vtable = &.{ .destroy = destroy };
-    if (info.code_size % @sizeOf(u32) != 0) return VkError.ValidationFailed;
-    const code = allocator.dupe(u32, info.p_code[0 .. info.code_size / @sizeOf(u32)]) catch return VkError.OutOfHostMemory;
-    errdefer allocator.free(code);
 
     self.* = .{
         .interface = interface,
-        .code = code,
         .ref_count = std.atomic.Value(usize).init(1),
     };
     return self;
@@ -35,7 +31,7 @@ pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
 }
 
 pub fn drop(self: *Self, allocator: std.mem.Allocator) void {
-    allocator.free(self.code);
+    self.interface.deinit();
     allocator.destroy(self);
 }
 
@@ -44,7 +40,6 @@ pub fn ref(self: *Self) void {
 }
 
 pub fn unref(self: *Self, allocator: std.mem.Allocator) void {
-    if (self.ref_count.fetchSub(1, .release) == 1) {
+    if (self.ref_count.fetchSub(1, .acq_rel) == 1)
         self.drop(allocator);
-    }
 }

src/intel/compiler/compiler.zig

@@ -0,0 +1,89 @@
+//! Flint-specific shader IR for Intel Gen hardware.
+//! This is the mutable, non-SSA layer between the common shader IR and machine code.
+
+pub const device = @import("device.zig");
+pub const ir = @import("ir/ir.zig");
+pub const lower = @import("lower/lower.zig");
+pub const targets = @import("targets/targets.zig");
+
+pub const Builder = ir.Builder;
+pub const id = ir.id;
+pub const instruction = ir.instruction;
+pub const operand = ir.operand;
+pub const printer = ir.printer;
+pub const program = ir.program;
+pub const pseudo = ir.pseudo;
+pub const validator = ir.validator;
+
+pub const Program = ir.Program;
+
+const std = @import("std");
+
+test "[ir] basic compute shader" {
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+
+    var shader = Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer shader.deinit();
+    var builder = Builder.init(&shader);
+
+    const value = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+        .name = "value",
+    });
+    const storage = try builder.addStorageBuffer(.{ .set = 0, .binding = 1, .name = "storage" });
+    const entry = try builder.addBlock("entry");
+    try builder.setEntryBlock(entry);
+
+    _ = try builder.appendInstruction(entry, .simd8, null, .{
+        .load_global_invocation_id = .{
+            .destination = .{ .register = .{ .virtual = value }, .type = .u32 },
+            .component = 0,
+        },
+    });
+    _ = try builder.appendInstruction(entry, .simd8, null, .{
+        .store_buffer = .{
+            .buffer = .{ .logical = storage },
+            .byte_offset = .{
+                .register = .{ .immediate = .{ .u32 = 0 } },
+                .type = .u32,
+                .region = operand.Region.broadcast(),
+            },
+            .source = .{
+                .register = .{ .virtual = value },
+                .type = .u32,
+                .region = operand.Region.contiguous(.simd8),
+            },
+        },
+    });
+    try builder.setTerminator(entry, .end_thread);
+    try validator.validate(&shader);
+
+    const text = try printer.allocPrint(std.testing.allocator, &shader);
+    defer std.testing.allocator.free(text);
+    try std.testing.expect(std.mem.indexOf(u8, text, "Flint compute program") != null);
+    try std.testing.expect(std.mem.indexOf(u8, text, "@storage = storage_buffer[set(0), binding(1)]") != null);
+    try std.testing.expect(std.mem.indexOf(u8, text, "load_global_invocation_id %value:u32, component(0)") != null);
+    try std.testing.expect(std.mem.indexOf(u8, text, "store_buffer @storage, 0:u32, %value:u32") != null);
+}
+
+test "[ir] ID stability after removal" {
+    var store: id.Store(id.VirtualFlagId, operand.VirtualFlag) = .{};
+    defer store.entries.deinit(std.testing.allocator);
+
+    const first = try store.add(std.testing.allocator, .{ .name = "first" });
+    try std.testing.expect(store.remove(first));
+    const second = try store.add(std.testing.allocator, .{ .name = "second" });
+
+    try std.testing.expect(first != second);
+    try std.testing.expect(store.get(first) == null);
+    try std.testing.expectEqualStrings("second", store.get(second).?.name.?);
+}

src/intel/compiler/device.zig

@@ -0,0 +1,138 @@
+pub const Generation = enum {
+    gen9,
+    gen10,
+    gen11,
+};
+
+pub const Platform = enum {
+    skylake,
+    broxton,
+    kabylake,
+    gemini_lake,
+    coffee_lake,
+    whiskey_lake,
+    comet_lake,
+    ice_lake,
+    elkhart_lake,
+    jasper_lake,
+};
+
+pub const DeviceInfo = struct {
+    generation: Generation,
+    platform: Platform,
+    pci_device_id: u16,
+
+    grf_count: u16,
+    grf_size_bytes: u16 = 32,
+
+    supports_int64: bool = false,
+    supports_float64: bool = false,
+    supports_half_float: bool = false,
+    supports_simd16: bool = false,
+    supports_simd32: bool = false,
+
+    pub fn supportsDispatch(self: DeviceInfo, width: DispatchWidth) bool {
+        return switch (width) {
+            .simd8 => true,
+            .simd16 => self.supports_simd16,
+            .simd32 => self.supports_simd32,
+        };
+    }
+
+    pub fn fromPciDeviceId(raw_pci_device_id: u32) ?DeviceInfo {
+        if (raw_pci_device_id > 0xffff)
+            return null;
+        const pci_device_id: u16 = @intCast(raw_pci_device_id);
+
+        const platform: Platform = switch (pci_device_id & 0xff00) {
+            0x1900 => .skylake,
+            0x5900 => .kabylake,
+
+            0x3e00 => switch (pci_device_id) {
+                0x3ea0,
+                0x3ea1,
+                0x3ea2,
+                0x3ea3,
+                0x3ea4,
+                => .whiskey_lake,
+
+                else => .coffee_lake,
+            },
+
+            0x9b00 => .comet_lake,
+            0x8a00 => .ice_lake,
+            0x4500 => .elkhart_lake,
+            0x4e00 => .jasper_lake,
+
+            else => switch (pci_device_id) {
+                0x0a84,
+                0x1a84,
+                0x1a85,
+                0x5a84,
+                0x5a85,
+                => .broxton,
+
+                0x3184,
+                0x3185,
+                => .gemini_lake,
+
+                0x87c0,
+                0x87ca,
+                => .kabylake,
+
+                else => return null,
+            },
+        };
+
+        const generation: Generation = switch (platform) {
+            .skylake,
+            .broxton,
+            .kabylake,
+            .gemini_lake,
+            .coffee_lake,
+            .whiskey_lake,
+            .comet_lake,
+            => .gen9,
+
+            .ice_lake,
+            .elkhart_lake,
+            .jasper_lake,
+            => .gen11,
+        };
+
+        return .{
+            .generation = generation,
+            .platform = platform,
+            .pci_device_id = pci_device_id,
+            .grf_count = 128,
+        };
+    }
+};
+
+pub const DispatchWidth = enum(u8) {
+    simd8 = 8,
+    simd16 = 16,
+    simd32 = 32,
+};
+
+pub const ExecutionSize = enum(u8) {
+    simd1 = 1,
+    simd2 = 2,
+    simd4 = 4,
+    simd8 = 8,
+    simd16 = 16,
+    simd32 = 32,
+};
+
+test "compiler device: classify supported Intel PCI IDs" {
+    const std = @import("std");
+
+    try std.testing.expectEqual(Platform.skylake, DeviceInfo.fromPciDeviceId(0x1912).?.platform);
+    try std.testing.expectEqual(Platform.broxton, DeviceInfo.fromPciDeviceId(0x5a84).?.platform);
+    try std.testing.expectEqual(Platform.kabylake, DeviceInfo.fromPciDeviceId(0x5916).?.platform);
+    try std.testing.expectEqual(Platform.whiskey_lake, DeviceInfo.fromPciDeviceId(0x3ea0).?.platform);
+    try std.testing.expectEqual(Platform.comet_lake, DeviceInfo.fromPciDeviceId(0x9bc5).?.platform);
+    try std.testing.expectEqual(Generation.gen11, DeviceInfo.fromPciDeviceId(0x8a52).?.generation);
+    try std.testing.expectEqual(Generation.gen11, DeviceInfo.fromPciDeviceId(0x4e55).?.generation);
+    try std.testing.expectEqual(@as(?DeviceInfo, null), DeviceInfo.fromPciDeviceId(0x46a6));
+}

src/intel/compiler/ir/Builder.zig

@@ -0,0 +1,190 @@
+const std = @import("std");
+const device = @import("../device.zig");
+const ids = @import("id.zig");
+const instruction = @import("instruction.zig");
+const operand = @import("operand.zig");
+const program_ir = @import("program.zig");
+const pseudo = @import("pseudo.zig");
+
+const Self = @This();
+
+pub const Error = std.mem.Allocator.Error || error{
+    InvalidBlock,
+    InvalidInstruction,
+    InvalidInsertionIndex,
+    TerminatorAlreadySet,
+};
+
+program: *program_ir.Program,
+
+pub fn init(program: *program_ir.Program) Self {
+    return .{ .program = program };
+}
+
+pub fn addVirtualRegister(self: *Self, register: operand.VirtualRegister) Error!ids.VirtualRegisterId {
+    return self.program.addVirtualRegister(register);
+}
+
+pub fn addVirtualFlag(self: *Self, flag: operand.VirtualFlag) Error!ids.VirtualFlagId {
+    return self.program.addVirtualFlag(flag);
+}
+
+pub fn addStorageBuffer(self: *Self, buffer: program_ir.StorageBuffer) Error!ids.StorageBufferId {
+    return self.program.addStorageBuffer(buffer);
+}
+
+pub fn addBlock(self: *Self, name: ?[]const u8) Error!ids.BlockId {
+    return self.program.addBlock(name);
+}
+
+pub fn setEntryBlock(self: *Self, block_id: ids.BlockId) Error!void {
+    return self.program.setEntryBlock(block_id);
+}
+
+pub fn addBlockParameter(self: *Self, block_id: ids.BlockId, parameter: pseudo.BlockParameter) Error!void {
+    const block = self.program.blocks.getMut(block_id) orelse return Error.InvalidBlock;
+    try block.parameters.append(self.program.allocator(), parameter);
+}
+
+pub fn clearBlockParameters(self: *Self, block_id: ids.BlockId) Error!void {
+    const block = self.program.blocks.getMut(block_id) orelse return Error.InvalidBlock;
+    block.parameters.clearRetainingCapacity();
+}
+
+pub fn edge(self: *Self, target: ids.BlockId, arguments: []const pseudo.EdgeArgument) Error!instruction.Edge {
+    if (!self.program.blocks.isLive(target))
+        return Error.InvalidBlock;
+    return .{
+        .target = target,
+        .arguments = try self.program.allocator().dupe(pseudo.EdgeArgument, arguments),
+    };
+}
+
+pub fn appendInstruction(self: *Self, block_id: ids.BlockId, execution_size: device.ExecutionSize, predicate: ?operand.Predicate, operation: instruction.Operation) Error!ids.InstructionId {
+    const block = self.program.blocks.get(block_id) orelse return Error.InvalidBlock;
+    return self.insertInstruction(block_id, block.instructions.items.len, execution_size, predicate, operation);
+}
+
+pub fn insertInstruction(
+    self: *Self,
+    block_id: ids.BlockId,
+    index: usize,
+    execution_size: device.ExecutionSize,
+    predicate: ?operand.Predicate,
+    operation: instruction.Operation,
+) Error!ids.InstructionId {
+    const block = self.program.blocks.getMut(block_id) orelse return Error.InvalidBlock;
+    if (index > block.instructions.items.len)
+        return Error.InvalidInsertionIndex;
+
+    const owned_operation = try instruction.cloneOperation(self.program.allocator(), operation);
+    const instruction_id = try self.program.instructions.add(self.program.allocator(), .{
+        .parent_block = block_id,
+        .execution_size = execution_size,
+        .predicate = predicate,
+        .operation = owned_operation,
+    });
+    errdefer std.debug.assert(self.program.instructions.remove(instruction_id));
+
+    try block.instructions.insert(self.program.allocator(), index, instruction_id);
+    return instruction_id;
+}
+
+pub fn replaceOperation(self: *Self, instruction_id: ids.InstructionId, operation: instruction.Operation) Error!void {
+    const inst = self.program.instructions.getMut(instruction_id) orelse return Error.InvalidInstruction;
+    const owned_operation = try instruction.cloneOperation(self.program.allocator(), operation);
+    inst.operation = owned_operation;
+}
+
+pub fn setStructuredControl(self: *Self, block_id: ids.BlockId, control: instruction.StructuredControl) Error!void {
+    const block = self.program.blocks.getMut(block_id) orelse return Error.InvalidBlock;
+    block.structured_control = control;
+}
+
+pub fn setTerminator(self: *Self, block_id: ids.BlockId, terminator: instruction.Terminator) Error!void {
+    return self.program.setTerminator(block_id, terminator);
+}
+
+pub fn replaceTerminator(self: *Self, block_id: ids.BlockId, terminator: instruction.Terminator) Error!void {
+    const block = self.program.blocks.getMut(block_id) orelse return Error.InvalidBlock;
+    block.terminator = try instruction.cloneTerminator(self.program.allocator(), terminator);
+}
+
+fn moveImmediate(register_id: ids.VirtualRegisterId, value: u32) instruction.Operation {
+    return .{
+        .move = .{
+            .destination = .{
+                .register = .{ .virtual = register_id },
+                .type = .u32,
+            },
+            .source = .{
+                .register = .{ .immediate = .{ .u32 = value } },
+                .type = .u32,
+                .region = operand.Region.broadcast(),
+            },
+        },
+    };
+}
+
+test "[ir] Builder: construction and ordered insertion" {
+    const validator = @import("validator.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+    var builder = Self.init(&program);
+
+    var register_name = [_]u8{ 'v', 'a', 'l', 'u', 'e' };
+    const register_id = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+        .name = &register_name,
+    });
+    register_name[0] = 'x';
+    try std.testing.expectEqualStrings("value", program.virtual_registers.get(register_id).?.name.?);
+
+    const flag_id = try builder.addVirtualFlag(.{ .name = "condition" });
+    try std.testing.expectEqualStrings("condition", program.virtual_flags.get(flag_id).?.name.?);
+
+    const entry = try builder.addBlock("entry");
+    const exit = try builder.addBlock("exit");
+    try builder.setEntryBlock(entry);
+
+    const second = try builder.appendInstruction(entry, .simd8, null, moveImmediate(register_id, 2));
+    const first = try builder.insertInstruction(entry, 0, .simd8, null, moveImmediate(register_id, 1));
+
+    const entry_block = program.blocks.get(entry).?;
+    try std.testing.expectEqualSlices(ids.InstructionId, &.{ first, second }, entry_block.instructions.items);
+    try std.testing.expectEqual(entry, program.instructions.get(first).?.parent_block);
+    try std.testing.expectEqual(entry, program.instructions.get(second).?.parent_block);
+
+    try builder.replaceOperation(first, moveImmediate(register_id, 3));
+    const replaced = program.instructions.get(first).?;
+    try std.testing.expectEqual(entry, replaced.parent_block);
+    try std.testing.expectEqual(device.ExecutionSize.simd8, replaced.execution_size);
+    try std.testing.expectEqual(@as(u32, 3), replaced.operation.move.source.register.immediate.u32);
+    try std.testing.expectError(Error.InvalidInstruction, builder.replaceOperation(ids.InstructionId.fromIndex(999), moveImmediate(register_id, 4)));
+
+    try builder.setStructuredControl(entry, .{ .selection = .{ .merge_block = exit } });
+    try builder.setTerminator(entry, .{ .jump = try builder.edge(exit, &.{}) });
+    try builder.setTerminator(exit, .end_thread);
+    try std.testing.expectError(Error.TerminatorAlreadySet, builder.setTerminator(entry, .end_thread));
+
+    const instruction_count = program.instructions.entries.items.len;
+    try std.testing.expectError(
+        Error.InvalidInsertionIndex,
+        builder.insertInstruction(entry, 3, .simd8, null, moveImmediate(register_id, 3)),
+    );
+    try std.testing.expectEqual(instruction_count, program.instructions.entries.items.len);
+
+    try validator.validate(&program);
+}

src/intel/compiler/ir/id.zig

@@ -0,0 +1,16 @@
+const shared_ids = @import("shader_ir").ir.id;
+
+pub const BlockTag = opaque {};
+pub const InstructionTag = opaque {};
+pub const VirtualRegisterTag = opaque {};
+pub const VirtualFlagTag = opaque {};
+pub const StorageBufferTag = opaque {};
+
+pub const BlockId = shared_ids.Id(BlockTag);
+pub const InstructionId = shared_ids.Id(InstructionTag);
+pub const VirtualRegisterId = shared_ids.Id(VirtualRegisterTag);
+pub const VirtualFlagId = shared_ids.Id(VirtualFlagTag);
+pub const StorageBufferId = shared_ids.Id(StorageBufferTag);
+
+pub const Id = shared_ids.Id;
+pub const Store = shared_ids.Store;

src/intel/compiler/ir/instruction.zig

@@ -0,0 +1,202 @@
+const std = @import("std");
+const device = @import("../device.zig");
+const ids = @import("id.zig");
+const operand = @import("operand.zig");
+const pseudo = @import("pseudo.zig");
+
+pub const LoadGlobalInvocationId = struct {
+    destination: operand.Destination,
+    component: u8,
+};
+
+pub const BufferReference = union(enum) {
+    logical: ids.StorageBufferId,
+    binding_table: u8,
+};
+
+pub const LoadBuffer = struct {
+    destination: operand.Destination,
+    buffer: BufferReference,
+    byte_offset: operand.Source,
+    immediate_offset: u32 = 0,
+};
+
+pub const StoreBuffer = struct {
+    buffer: BufferReference,
+    byte_offset: operand.Source,
+    immediate_offset: u32 = 0,
+    source: operand.Source,
+};
+
+pub const ArrayLength = struct {
+    destination: operand.Destination,
+    buffer: BufferReference,
+    byte_offset: operand.Source,
+    stride: u32,
+};
+
+pub const SurfaceRead = struct {
+    destination: operand.Destination,
+    binding_table: u8,
+    address: operand.Source,
+    immediate_offset: u32 = 0,
+};
+
+pub const SurfaceWrite = struct {
+    binding_table: u8,
+    address: operand.Source,
+    immediate_offset: u32 = 0,
+    data: operand.Source,
+};
+
+pub const SurfaceMessageKind = enum {
+    read,
+    write,
+};
+
+pub const SurfaceMessage = struct {
+    kind: SurfaceMessageKind,
+    binding_table: u8,
+    payload: operand.RegisterSpan,
+    response: ?operand.RegisterSpan,
+    data_type: operand.DataType,
+};
+
+pub const Move = struct {
+    destination: operand.Destination,
+    source: operand.Source,
+};
+
+pub const BinaryOpcode = enum {
+    add,
+    multiply,
+    bitwise_and,
+    bitwise_or,
+    bitwise_xor,
+    shift_left,
+    shift_right,
+};
+
+pub const Binary = struct {
+    opcode: BinaryOpcode,
+    destination: operand.Destination,
+    lhs: operand.Source,
+    rhs: operand.Source,
+};
+
+pub const CompareOpcode = enum {
+    equal,
+    not_equal,
+    less_than,
+    less_or_equal,
+    greater_than,
+    greater_or_equal,
+};
+
+pub const Compare = struct {
+    opcode: CompareOpcode,
+    destination: operand.FlagRef,
+    lhs: operand.Source,
+    rhs: operand.Source,
+};
+
+pub const MathOpcode = enum {
+    integer_quotient,
+};
+
+pub const Math = struct {
+    opcode: MathOpcode,
+    destination: operand.Destination,
+    lhs: operand.Source,
+    rhs: operand.Source,
+};
+
+pub const Operation = union(enum) {
+    load_global_invocation_id: LoadGlobalInvocationId,
+    load_buffer: LoadBuffer,
+    store_buffer: StoreBuffer,
+    array_length: ArrayLength,
+    surface_read: SurfaceRead,
+    surface_write: SurfaceWrite,
+    surface_message: SurfaceMessage,
+    move: Move,
+    binary: Binary,
+    math: Math,
+    compare: Compare,
+    parallel_copy: pseudo.ParallelCopy,
+};
+
+pub fn cloneOperation(allocator: std.mem.Allocator, operation: Operation) std.mem.Allocator.Error!Operation {
+    return switch (operation) {
+        .parallel_copy => |copy| .{
+            .parallel_copy = .{
+                .register_copies = try allocator.dupe(pseudo.RegisterCopy, copy.register_copies),
+                .flag_copies = try allocator.dupe(pseudo.FlagCopy, copy.flag_copies),
+            },
+        },
+        else => operation,
+    };
+}
+
+pub const Instruction = struct {
+    parent_block: ids.BlockId,
+    execution_size: device.ExecutionSize,
+    predicate: ?operand.Predicate = null,
+    operation: Operation,
+};
+
+pub const Edge = struct {
+    target: ids.BlockId,
+    arguments: []const pseudo.EdgeArgument,
+};
+
+pub const Terminator = union(enum) {
+    jump: Edge,
+    conditional_branch: struct {
+        predicate: operand.Predicate,
+        true_edge: Edge,
+        false_edge: Edge,
+    },
+    end_thread,
+    @"unreachable",
+};
+
+pub fn cloneEdge(allocator: std.mem.Allocator, edge: Edge) std.mem.Allocator.Error!Edge {
+    return .{
+        .target = edge.target,
+        .arguments = try allocator.dupe(pseudo.EdgeArgument, edge.arguments),
+    };
+}
+
+pub fn cloneTerminator(allocator: std.mem.Allocator, terminator: Terminator) std.mem.Allocator.Error!Terminator {
+    return switch (terminator) {
+        .jump => |edge| .{ .jump = try cloneEdge(allocator, edge) },
+        .conditional_branch => |branch| .{
+            .conditional_branch = .{
+                .predicate = branch.predicate,
+                .true_edge = try cloneEdge(allocator, branch.true_edge),
+                .false_edge = try cloneEdge(allocator, branch.false_edge),
+            },
+        },
+        else => terminator,
+    };
+}
+
+pub const StructuredControl = union(enum) {
+    none,
+    selection: struct {
+        merge_block: ids.BlockId,
+    },
+    loop: struct {
+        merge_block: ids.BlockId,
+        continue_block: ids.BlockId,
+    },
+};
+
+pub const Block = struct {
+    parameters: std.ArrayList(pseudo.BlockParameter) = .empty,
+    instructions: std.ArrayList(ids.InstructionId) = .empty,
+    terminator: ?Terminator = null,
+    structured_control: StructuredControl = .none,
+    name: ?[]const u8 = null,
+};

src/intel/compiler/ir/ir.zig

@@ -0,0 +1,10 @@
+pub const Builder = @import("Builder.zig");
+pub const id = @import("id.zig");
+pub const instruction = @import("instruction.zig");
+pub const operand = @import("operand.zig");
+pub const printer = @import("printer.zig");
+pub const program = @import("program.zig");
+pub const pseudo = @import("pseudo.zig");
+pub const validator = @import("validator.zig");
+
+pub const Program = program.Program;

src/intel/compiler/ir/operand.zig

@@ -0,0 +1,147 @@
+const device = @import("../device.zig");
+const ids = @import("id.zig");
+
+pub const DataType = enum {
+    u8,
+    i8,
+    u16,
+    i16,
+    f16,
+    u32,
+    i32,
+    f32,
+    u64,
+    i64,
+    f64,
+
+    pub fn sizeBytes(self: DataType) u8 {
+        return switch (self) {
+            .u8, .i8 => 1,
+            .u16, .i16, .f16 => 2,
+            .u32, .i32, .f32 => 4,
+            .u64, .i64, .f64 => 8,
+        };
+    }
+
+    pub fn isInitialTargetType(self: DataType) bool {
+        return switch (self) {
+            .u32, .i32, .f32 => true,
+            else => false,
+        };
+    }
+};
+
+pub const RegisterClass = enum {
+    uniform,
+    payload,
+    response,
+    temporary,
+};
+
+pub const VirtualRegister = struct {
+    size_bytes: u32,
+    alignment_bytes: u16,
+    element_type: DataType,
+    lane_count: u8,
+    class: RegisterClass,
+    spillable: bool = true,
+    name: ?[]const u8 = null,
+};
+
+pub const VirtualFlag = struct {
+    name: ?[]const u8 = null,
+};
+
+pub const PhysicalGrf = struct {
+    number: u16,
+    byte_offset: u8 = 0,
+};
+
+pub const PhysicalFlag = struct {
+    register: u8 = 0,
+    subregister: u8 = 0,
+};
+
+pub const ArchitectureRegister = union(enum) {
+    flag: u8,
+    address: u8,
+    accumulator: u8,
+    notification: u8,
+    instruction_pointer,
+};
+
+pub const Immediate = union(enum) {
+    u32: u32,
+    i32: i32,
+    f32: f32,
+};
+
+pub const RegisterRef = union(enum) {
+    virtual: ids.VirtualRegisterId,
+    physical_grf: PhysicalGrf,
+    architecture: ArchitectureRegister,
+    immediate: Immediate,
+    null,
+};
+
+pub const FlagRef = union(enum) {
+    virtual: ids.VirtualFlagId,
+    physical: PhysicalFlag,
+};
+
+pub const Predicate = struct {
+    flag: FlagRef,
+    inverse: bool = false,
+};
+
+pub const Region = struct {
+    byte_offset: u16 = 0,
+    vertical_stride: u8,
+    width: u8,
+    horizontal_stride: u8,
+
+    pub fn scalar() Region {
+        return .{
+            .vertical_stride = 0,
+            .width = 1,
+            .horizontal_stride = 0,
+        };
+    }
+
+    pub fn contiguous(execution_size: device.ExecutionSize) Region {
+        const width: u8 = @intFromEnum(execution_size);
+        return .{
+            .vertical_stride = width,
+            .width = width,
+            .horizontal_stride = 1,
+        };
+    }
+
+    pub fn broadcast() Region {
+        return scalar();
+    }
+};
+
+pub const DestinationRegion = struct {
+    byte_offset: u16 = 0,
+    horizontal_stride: u8 = 1,
+};
+
+pub const Source = struct {
+    register: RegisterRef,
+    type: DataType,
+    region: Region,
+    negate: bool = false,
+    absolute: bool = false,
+};
+
+pub const Destination = struct {
+    register: RegisterRef,
+    type: DataType,
+    region: DestinationRegion = .{},
+};
+
+pub const RegisterSpan = struct {
+    base: RegisterRef,
+    register_count: u8,
+};

src/intel/compiler/ir/printer.zig

@@ -0,0 +1,450 @@
+const std = @import("std");
+const device = @import("../device.zig");
+const ids = @import("id.zig");
+const inst_ir = @import("instruction.zig");
+const operand = @import("operand.zig");
+const program_ir = @import("program.zig");
+const pseudo = @import("pseudo.zig");
+
+const indent = "    ";
+
+pub fn write(program: *const program_ir.Program, writer: *std.Io.Writer) std.Io.Writer.Error!void {
+    try writer.writeAll("; Flint compute program:\n");
+    try writer.print(";   .workgroup_size: [{d}, {d}, {d}]\n", .{ program.workgroup_size[0], program.workgroup_size[1], program.workgroup_size[2] });
+    try writer.print(";   .generation: {t}\n", .{program.device_info.generation});
+    try writer.print(";   .platform: {t}\n", .{program.device_info.platform});
+    try writer.print(";   .dispatch_width: {t}\n\n", .{program.dispatch_width});
+
+    for (program.storage_buffers.entries.items, 0..) |entry, index| {
+        const buffer = entry orelse continue;
+        try writeStorageBufferRef(program, writer, ids.StorageBufferId.fromIndex(index));
+        try writer.print(" = storage_buffer[set({d}), binding({d})]\n", .{ buffer.set, buffer.binding });
+    }
+    if (program.storage_buffers.entries.items.len != 0)
+        try writer.writeByte('\n');
+
+    for (program.virtual_registers.entries.items, 0..) |entry, index| {
+        const register = entry orelse continue;
+        try writeVirtualRegisterRef(program, writer, ids.VirtualRegisterId.fromIndex(index));
+        try writer.print(": vgrf {t}[{d}], class({t}), size({d}), alignment({d}){s}\n", .{
+            register.element_type,
+            register.lane_count,
+            register.class,
+            register.size_bytes,
+            register.alignment_bytes,
+            if (register.spillable) ", spillable" else "",
+        });
+    }
+
+    for (program.virtual_flags.entries.items, 0..) |entry, index| {
+        _ = entry orelse continue;
+        try writeVirtualFlagRef(program, writer, ids.VirtualFlagId.fromIndex(index));
+        try writer.writeAll(": vflag\n");
+    }
+
+    try writer.writeByte('\n');
+
+    for (program.blocks.entries.items, 0..) |entry, block_index| {
+        const block = entry orelse continue;
+        const block_id = ids.BlockId.fromIndex(block_index);
+
+        try writeBlockRef(program, writer, block_id);
+        if (block.parameters.items.len != 0) {
+            try writer.writeByte('(');
+            for (block.parameters.items, 0..) |parameter, index| {
+                if (index != 0)
+                    try writer.writeAll(", ");
+                try writeBlockParameter(program, writer, parameter);
+            }
+            try writer.writeByte(')');
+        }
+        try writer.writeAll(":\n");
+
+        switch (block.structured_control) {
+            .none => {},
+            .selection => |selection| {
+                try writer.writeAll(indent ++ "structured_selection ");
+                try writeBlockRef(program, writer, selection.merge_block);
+                try writer.writeByte('\n');
+            },
+            .loop => |loop| {
+                try writer.writeAll(indent ++ "structured_loop merge(");
+                try writeBlockRef(program, writer, loop.merge_block);
+                try writer.writeAll("), continue(");
+                try writeBlockRef(program, writer, loop.continue_block);
+                try writer.writeAll(")\n");
+            },
+        }
+
+        for (block.instructions.items) |instruction_id| {
+            const instruction = program.instructions.get(instruction_id) orelse continue;
+            try writer.writeAll(indent);
+            try writeInstruction(program, writer, instruction.*);
+            try writer.writeByte('\n');
+        }
+
+        if (block.terminator) |terminator| {
+            try writer.writeAll(indent);
+            try writeTerminator(program, writer, terminator);
+            try writer.writeAll("\n\n");
+        } else {
+            try writer.writeAll(indent ++ "<missing terminator>\n\n");
+        }
+    }
+}
+
+pub fn allocPrint(allocator: std.mem.Allocator, program: *const program_ir.Program) ![]u8 {
+    var output: std.Io.Writer.Allocating = .init(allocator);
+    defer output.deinit();
+    try write(program, &output.writer);
+    return output.toOwnedSlice();
+}
+
+fn writeInstruction(program: *const program_ir.Program, writer: *std.Io.Writer, instruction: inst_ir.Instruction) !void {
+    try writer.print("[simd{d}] ", .{@intFromEnum(instruction.execution_size)});
+    if (instruction.predicate) |predicate| {
+        try writePredicate(program, writer, predicate);
+        try writer.writeByte(' ');
+    }
+    try writeOperation(program, writer, instruction.execution_size, instruction.operation);
+}
+
+fn writeOperation(program: *const program_ir.Program, writer: *std.Io.Writer, execution_size: device.ExecutionSize, operation: inst_ir.Operation) !void {
+    switch (operation) {
+        .load_global_invocation_id => |op| {
+            try writer.writeAll("load_global_invocation_id ");
+            try writeDestination(program, writer, execution_size, op.destination);
+            try writer.print(", component({d})", .{op.component});
+        },
+        .load_buffer => |op| {
+            try writer.writeAll("load_buffer ");
+            try writeDestination(program, writer, execution_size, op.destination);
+            try writer.writeAll(", ");
+            try writeBufferReference(program, writer, op.buffer);
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.byte_offset);
+            if (op.immediate_offset != 0)
+                try writer.print(", offset({d})", .{op.immediate_offset});
+        },
+        .store_buffer => |op| {
+            try writer.writeAll("store_buffer ");
+            try writeBufferReference(program, writer, op.buffer);
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.byte_offset);
+            if (op.immediate_offset != 0)
+                try writer.print(", offset({d})", .{op.immediate_offset});
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.source);
+        },
+        .array_length => |op| {
+            try writer.writeAll("array_length ");
+            try writeDestination(program, writer, execution_size, op.destination);
+            try writer.writeAll(", ");
+            try writeBufferReference(program, writer, op.buffer);
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.byte_offset);
+            try writer.print(", stride({d})", .{op.stride});
+        },
+        .surface_read => |op| {
+            try writer.writeAll("surface_read ");
+            try writeDestination(program, writer, execution_size, op.destination);
+            try writer.print(", bti({d}), ", .{op.binding_table});
+            try writeSource(program, writer, execution_size, op.address);
+            if (op.immediate_offset != 0)
+                try writer.print(", offset({d})", .{op.immediate_offset});
+        },
+        .surface_write => |op| {
+            try writer.print("surface_write bti({d}), ", .{op.binding_table});
+            try writeSource(program, writer, execution_size, op.address);
+            if (op.immediate_offset != 0)
+                try writer.print(", offset({d})", .{op.immediate_offset});
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.data);
+        },
+        .surface_message => |op| {
+            try writer.print("surface_message {t} bti({d}), payload(", .{ op.kind, op.binding_table });
+            try writeRegister(program, writer, op.payload.base);
+            try writer.print(", {d})", .{op.payload.register_count});
+            if (op.response) |response| {
+                try writer.writeAll(", response(");
+                try writeRegister(program, writer, response.base);
+                try writer.print(", {d})", .{response.register_count});
+            }
+            try writer.print(", type({t})", .{op.data_type});
+        },
+        .move => |op| {
+            try writer.writeAll("mov ");
+            try writeDestination(program, writer, execution_size, op.destination);
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.source);
+        },
+        .binary => |op| {
+            try writer.print("{t} ", .{op.opcode});
+            try writeDestination(program, writer, execution_size, op.destination);
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.lhs);
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.rhs);
+        },
+        .compare => |op| {
+            try writer.print("cmp_{t} ", .{op.opcode});
+            try writeFlagRef(program, writer, op.destination);
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.lhs);
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.rhs);
+        },
+        .math => |op| {
+            try writer.print("{t} ", .{op.opcode});
+            try writeDestination(program, writer, execution_size, op.destination);
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.lhs);
+            try writer.writeAll(", ");
+            try writeSource(program, writer, execution_size, op.rhs);
+        },
+        .parallel_copy => |op| try writeParallelCopy(program, writer, execution_size, op),
+    }
+}
+
+fn writeParallelCopy(program: *const program_ir.Program, writer: *std.Io.Writer, execution_size: device.ExecutionSize, copy: pseudo.ParallelCopy) !void {
+    try writer.writeAll("parallel_copy [");
+    var needs_separator = false;
+
+    for (copy.register_copies) |item| {
+        if (needs_separator)
+            try writer.writeAll(", ");
+        try writeDestination(program, writer, execution_size, item.destination);
+        try writer.writeAll(" <- ");
+        try writeSource(program, writer, execution_size, item.source);
+        needs_separator = true;
+    }
+
+    for (copy.flag_copies) |item| {
+        if (needs_separator)
+            try writer.writeAll(", ");
+        try writeVirtualFlagRef(program, writer, item.destination);
+        try writer.writeAll(" <- ");
+        switch (item.source) {
+            .constant => |value| try writer.writeAll(if (value) "true" else "false"),
+            .dynamic => |predicate| try writePredicate(program, writer, predicate),
+        }
+        needs_separator = true;
+    }
+
+    try writer.writeByte(']');
+}
+
+fn writeTerminator(program: *const program_ir.Program, writer: *std.Io.Writer, terminator: inst_ir.Terminator) !void {
+    switch (terminator) {
+        .jump => |edge| {
+            try writer.writeAll("jump ");
+            try writeEdge(program, writer, edge);
+        },
+        .conditional_branch => |branch| {
+            try writer.writeAll("conditional_branch ");
+            try writePredicate(program, writer, branch.predicate);
+            try writer.writeAll(", ");
+            try writeEdge(program, writer, branch.true_edge);
+            try writer.writeAll(", ");
+            try writeEdge(program, writer, branch.false_edge);
+        },
+        .end_thread => try writer.writeAll("end_thread"),
+        .@"unreachable" => try writer.writeAll("unreachable"),
+    }
+}
+
+fn writeBlockParameter(program: *const program_ir.Program, writer: *std.Io.Writer, parameter: pseudo.BlockParameter) !void {
+    switch (parameter) {
+        .register => |register_id| try writeVirtualRegisterRef(program, writer, register_id),
+        .flag => |flag_id| try writeVirtualFlagRef(program, writer, flag_id),
+    }
+}
+
+fn writeEdge(program: *const program_ir.Program, writer: *std.Io.Writer, edge: inst_ir.Edge) !void {
+    try writeBlockRef(program, writer, edge.target);
+    if (edge.arguments.len == 0)
+        return;
+
+    const execution_size: device.ExecutionSize = @enumFromInt(@intFromEnum(program.dispatch_width));
+    try writer.writeByte('(');
+    for (edge.arguments, 0..) |argument, index| {
+        if (index != 0)
+            try writer.writeAll(", ");
+        switch (argument) {
+            .source => |source| try writeSource(program, writer, execution_size, source),
+            .predicate => |predicate_value| switch (predicate_value) {
+                .constant => |value| try writer.writeAll(if (value) "true" else "false"),
+                .dynamic => |predicate| try writePredicate(program, writer, predicate),
+            },
+        }
+    }
+    try writer.writeByte(')');
+}
+
+fn writeSource(program: *const program_ir.Program, writer: *std.Io.Writer, execution_size: device.ExecutionSize, source: operand.Source) !void {
+    if (source.negate)
+        try writer.writeByte('-');
+    if (source.absolute)
+        try writer.writeAll("abs(");
+
+    try writeRegister(program, writer, source.register);
+    try writer.print(":{t}", .{source.type});
+    if (source.register != .immediate)
+        try writeSourceRegion(writer, execution_size, source.register, source.region);
+
+    if (source.absolute)
+        try writer.writeByte(')');
+}
+
+fn writeDestination(program: *const program_ir.Program, writer: *std.Io.Writer, execution_size: device.ExecutionSize, destination: operand.Destination) !void {
+    _ = execution_size;
+    try writeRegister(program, writer, destination.register);
+    try writer.print(":{t}", .{destination.type});
+    try writeDestinationRegion(writer, destination.register, destination.region);
+}
+
+fn writeSourceRegion(writer: *std.Io.Writer, execution_size: device.ExecutionSize, register: operand.RegisterRef, region: operand.Region) !void {
+    const byte_offset = registerByteOffset(register) + region.byte_offset;
+    const execution_width: u8 = @intFromEnum(execution_size);
+    const is_default = region.vertical_stride == execution_width and
+        region.width == execution_width and
+        region.horizontal_stride == 1;
+    const is_broadcast = region.vertical_stride == 0 and
+        region.width == 1 and
+        region.horizontal_stride == 0;
+
+    if (byte_offset == 0 and is_default)
+        return;
+
+    try writer.writeByte('[');
+    if (byte_offset != 0)
+        try writer.print("byte={d}", .{byte_offset});
+
+    if (is_broadcast) {
+        if (byte_offset != 0)
+            try writer.writeAll(", ");
+        try writer.writeAll("broadcast");
+    } else if (!is_default) {
+        if (byte_offset != 0)
+            try writer.writeAll(", ");
+        try writer.print("vstride={d}, width={d}, hstride={d}", .{
+            region.vertical_stride,
+            region.width,
+            region.horizontal_stride,
+        });
+    }
+    try writer.writeByte(']');
+}
+
+fn writeDestinationRegion(writer: *std.Io.Writer, register: operand.RegisterRef, region: operand.DestinationRegion) !void {
+    const byte_offset = registerByteOffset(register) + region.byte_offset;
+    if (byte_offset == 0 and region.horizontal_stride == 1)
+        return;
+
+    try writer.writeByte('[');
+    if (byte_offset != 0)
+        try writer.print("byte={d}", .{byte_offset});
+    if (region.horizontal_stride != 1) {
+        if (byte_offset != 0)
+            try writer.writeAll(", ");
+        try writer.print("hstride={d}", .{region.horizontal_stride});
+    }
+    try writer.writeByte(']');
+}
+
+fn registerByteOffset(register: operand.RegisterRef) u16 {
+    return switch (register) {
+        .physical_grf => |physical| physical.byte_offset,
+        else => 0,
+    };
+}
+
+fn writeRegister(program: *const program_ir.Program, writer: *std.Io.Writer, register: operand.RegisterRef) !void {
+    switch (register) {
+        .virtual => |virtual| try writeVirtualRegisterRef(program, writer, virtual),
+        .physical_grf => |physical| try writer.print("r{d}", .{physical.number}),
+        .architecture => |architecture| try writeArchitectureRegister(writer, architecture),
+        .immediate => |immediate| try writeImmediate(writer, immediate),
+        .null => try writer.writeAll("null"),
+    }
+}
+
+fn writeArchitectureRegister(writer: *std.Io.Writer, register: operand.ArchitectureRegister) !void {
+    switch (register) {
+        .flag => |index| try writer.print("f{d}", .{index}),
+        .address => |index| try writer.print("a{d}", .{index}),
+        .accumulator => |index| try writer.print("acc{d}", .{index}),
+        .notification => |index| try writer.print("n{d}", .{index}),
+        .instruction_pointer => try writer.writeAll("ip"),
+    }
+}
+
+fn writeImmediate(writer: *std.Io.Writer, immediate: operand.Immediate) !void {
+    switch (immediate) {
+        .u32 => |value| try writer.print("{d}", .{value}),
+        .i32 => |value| try writer.print("{d}", .{value}),
+        .f32 => |value| try writer.print("{d}", .{value}),
+    }
+}
+
+fn writePredicate(program: *const program_ir.Program, writer: *std.Io.Writer, predicate: operand.Predicate) !void {
+    try writer.writeAll(if (predicate.inverse) "(-" else "(+");
+    try writeFlagRef(program, writer, predicate.flag);
+    try writer.writeByte(')');
+}
+
+fn writeFlagRef(program: *const program_ir.Program, writer: *std.Io.Writer, flag: operand.FlagRef) !void {
+    switch (flag) {
+        .virtual => |virtual| try writeVirtualFlagRef(program, writer, virtual),
+        .physical => |physical| try writer.print("f{d}.{d}", .{ physical.register, physical.subregister }),
+    }
+}
+
+fn writeBufferReference(program: *const program_ir.Program, writer: *std.Io.Writer, reference: inst_ir.BufferReference) !void {
+    switch (reference) {
+        .logical => |buffer| try writeStorageBufferRef(program, writer, buffer),
+        .binding_table => |index| try writer.print("bti({d})", .{index}),
+    }
+}
+
+fn writeStorageBufferRef(program: *const program_ir.Program, writer: *std.Io.Writer, buffer_id: ids.StorageBufferId) !void {
+    const buffer = program.storage_buffers.get(buffer_id);
+    try writeNamedRef(writer, if (buffer) |value| value.name else null, "buffer", buffer_id.index(), '@');
+}
+
+fn writeVirtualRegisterRef(program: *const program_ir.Program, writer: *std.Io.Writer, register_id: ids.VirtualRegisterId) !void {
+    const register = program.virtual_registers.get(register_id);
+    try writeNamedRef(writer, if (register) |value| value.name else null, "v", register_id.index(), '%');
+}
+
+fn writeVirtualFlagRef(program: *const program_ir.Program, writer: *std.Io.Writer, flag_id: ids.VirtualFlagId) !void {
+    const flag = program.virtual_flags.get(flag_id);
+    try writeNamedRef(writer, if (flag) |value| value.name else null, "f", flag_id.index(), '%');
+}
+
+fn writeBlockRef(program: *const program_ir.Program, writer: *std.Io.Writer, block_id: ids.BlockId) !void {
+    const block = program.blocks.get(block_id);
+    try writeNamedRef(writer, if (block) |value| value.name else null, "b", block_id.index(), '.');
+}
+
+fn writeNamedRef(writer: *std.Io.Writer, name: ?[]const u8, fallback: []const u8, index: usize, prefix: u8) !void {
+    try writer.writeByte(prefix);
+    if (name) |text| {
+        if (isValidName(text)) {
+            try writer.writeAll(text);
+            return;
+        }
+    }
+    try writer.print("{s}{d}", .{ fallback, index });
+}
+
+fn isValidName(name: []const u8) bool {
+    if (name.len == 0 or (!std.ascii.isAlphabetic(name[0]) and name[0] != '_'))
+        return false;
+
+    for (name[1..]) |byte| {
+        if (!std.ascii.isAlphanumeric(byte) and byte != '_')
+            return false;
+    }
+    return true;
+}

src/intel/compiler/ir/program.zig

@@ -0,0 +1,150 @@
+const std = @import("std");
+
+const device = @import("../device.zig");
+const ids = @import("id.zig");
+const instructions = @import("instruction.zig");
+const operand = @import("operand.zig");
+
+pub const Properties = packed struct {
+    common_ir_lowered: bool = false,
+    instructions_selected: bool = false,
+    block_parameters_lowered: bool = false,
+    parallel_copies_lowered: bool = false,
+
+    compute_abi_lowered: bool = false,
+    system_values_lowered: bool = false,
+    resources_lowered: bool = false,
+    messages_lowered: bool = false,
+    message_addresses_lowered: bool = false,
+    message_payloads_lowered: bool = false,
+    control_flow_lowered: bool = false,
+
+    regions_legalized: bool = false,
+    types_legalized: bool = false,
+
+    registers_allocated: bool = false,
+    flags_allocated: bool = false,
+    branches_resolved: bool = false,
+
+    _padding: u16 = 0,
+};
+
+pub const StorageBuffer = struct {
+    set: u32,
+    binding: u32,
+    name: ?[]const u8 = null,
+};
+
+pub const PayloadLayout = struct {
+    header_grf: ?operand.PhysicalGrf = null,
+};
+
+pub const ProgramData = struct {
+    payload_grf_count: u16 = 0,
+    total_grf_count: u16 = 0,
+    scratch_size_bytes: u32 = 0,
+};
+
+pub const BlockStore = ids.Store(ids.BlockId, instructions.Block);
+pub const InstructionStore = ids.Store(ids.InstructionId, instructions.Instruction);
+pub const VirtualRegisterStore = ids.Store(ids.VirtualRegisterId, operand.VirtualRegister);
+pub const VirtualFlagStore = ids.Store(ids.VirtualFlagId, operand.VirtualFlag);
+pub const StorageBufferStore = ids.Store(ids.StorageBufferId, StorageBuffer);
+
+pub const Program = struct {
+    arena: std.heap.ArenaAllocator,
+
+    workgroup_size: [3]u32,
+    device_info: device.DeviceInfo,
+    dispatch_width: device.DispatchWidth,
+
+    entry_block: ?ids.BlockId = null,
+
+    blocks: BlockStore = .{},
+    instructions: InstructionStore = .{},
+    virtual_registers: VirtualRegisterStore = .{},
+    virtual_flags: VirtualFlagStore = .{},
+    storage_buffers: StorageBufferStore = .{},
+
+    payload: PayloadLayout = .{},
+    program_data: ProgramData = .{},
+    properties: Properties = .{},
+
+    pub fn init(backing_allocator: std.mem.Allocator, workgroup_size: [3]u32, device_info: device.DeviceInfo, dispatch_width: device.DispatchWidth) Program {
+        return .{
+            .arena = std.heap.ArenaAllocator.init(backing_allocator),
+            .workgroup_size = workgroup_size,
+            .device_info = device_info,
+            .dispatch_width = dispatch_width,
+        };
+    }
+
+    pub fn deinit(self: *Program) void {
+        self.arena.deinit();
+        self.* = undefined;
+    }
+
+    pub fn allocator(self: *Program) std.mem.Allocator {
+        return self.arena.allocator();
+    }
+
+    pub fn addVirtualRegister(self: *Program, register: operand.VirtualRegister) !ids.VirtualRegisterId {
+        var owned = register;
+        if (register.name) |name|
+            owned.name = try self.allocator().dupe(u8, name);
+        return self.virtual_registers.add(self.allocator(), owned);
+    }
+
+    pub fn addVirtualFlag(self: *Program, flag: operand.VirtualFlag) !ids.VirtualFlagId {
+        var owned = flag;
+        if (flag.name) |name|
+            owned.name = try self.allocator().dupe(u8, name);
+        return self.virtual_flags.add(self.allocator(), owned);
+    }
+
+    pub fn addStorageBuffer(self: *Program, buffer: StorageBuffer) !ids.StorageBufferId {
+        var owned = buffer;
+        if (buffer.name) |name|
+            owned.name = try self.allocator().dupe(u8, name);
+        return self.storage_buffers.add(self.allocator(), owned);
+    }
+
+    pub fn addBlock(self: *Program, name: ?[]const u8) !ids.BlockId {
+        const owned_name = if (name) |value| try self.allocator().dupe(u8, value) else null;
+        const block_id = try self.blocks.add(self.allocator(), .{
+            .name = owned_name,
+        });
+        if (self.entry_block == null)
+            self.entry_block = block_id;
+        return block_id;
+    }
+
+    pub fn setEntryBlock(self: *Program, block_id: ids.BlockId) !void {
+        if (!self.blocks.isLive(block_id))
+            return error.InvalidBlock;
+        self.entry_block = block_id;
+    }
+
+    pub fn appendInstruction(self: *Program, block_id: ids.BlockId, execution_size: device.ExecutionSize, predicate: ?operand.Predicate, operation: instructions.Operation) !ids.InstructionId {
+        const block = self.blocks.getMut(block_id) orelse return error.InvalidBlock;
+
+        const owned_operation = try instructions.cloneOperation(self.allocator(), operation);
+        const instruction_id = try self.instructions.add(self.allocator(), .{
+            .parent_block = block_id,
+            .execution_size = execution_size,
+            .predicate = predicate,
+            .operation = owned_operation,
+        });
+        errdefer std.debug.assert(self.instructions.remove(instruction_id));
+
+        try block.instructions.append(self.allocator(), instruction_id);
+        return instruction_id;
+    }
+
+    pub fn setTerminator(self: *Program, block_id: ids.BlockId, terminator: instructions.Terminator) !void {
+        const block = self.blocks.getMut(block_id) orelse return error.InvalidBlock;
+        if (block.terminator != null)
+            return error.TerminatorAlreadySet;
+        block.terminator = try instructions.cloneTerminator(self.allocator(), terminator);
+    }
+};

src/intel/compiler/ir/pseudo.zig

@@ -0,0 +1,183 @@
+const ids = @import("id.zig");
+const operand = @import("operand.zig");
+
+pub const PredicateValue = union(enum) {
+    constant: bool,
+    dynamic: operand.Predicate,
+};
+
+pub const BlockParameter = union(enum) {
+    register: ids.VirtualRegisterId,
+    flag: ids.VirtualFlagId,
+};
+
+pub const EdgeArgument = union(enum) {
+    source: operand.Source,
+    predicate: PredicateValue,
+};
+
+pub const RegisterCopy = struct {
+    destination: operand.Destination,
+    source: operand.Source,
+};
+
+pub const FlagCopy = struct {
+    destination: ids.VirtualFlagId,
+    source: PredicateValue,
+};
+
+/// A simultaneous assignment: every source is read before any destination is
+/// written. This pseudo-operation must be eliminated before machine emission.
+pub const ParallelCopy = struct {
+    register_copies: []const RegisterCopy,
+    flag_copies: []const FlagCopy,
+};
+
+test "[ir] pseudo: parallel copy ownership and printing" {
+    const std = @import("std");
+    const Builder = @import("Builder.zig");
+    const device = @import("../device.zig");
+    const printer = @import("printer.zig");
+    const program_ir = @import("program.zig");
+    const validator = @import("validator.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const source_register = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+        .name = "source",
+    });
+    const destination_register = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+        .name = "destination",
+    });
+    const source_flag = try builder.addVirtualFlag(.{ .name = "source_flag" });
+    const destination_flag = try builder.addVirtualFlag(.{ .name = "destination_flag" });
+    const entry = try builder.addBlock("entry");
+
+    var register_copies = [_]RegisterCopy{.{
+        .destination = .{
+            .register = .{ .virtual = destination_register },
+            .type = .u32,
+        },
+        .source = .{
+            .register = .{ .virtual = source_register },
+            .type = .u32,
+            .region = operand.Region.contiguous(.simd8),
+        },
+    }};
+    var flag_copies = [_]FlagCopy{.{
+        .destination = destination_flag,
+        .source = .{ .dynamic = .{ .flag = .{ .virtual = source_flag } } },
+    }};
+
+    const copy_id = try builder.appendInstruction(entry, .simd8, null, .{
+        .parallel_copy = .{
+            .register_copies = &register_copies,
+            .flag_copies = &flag_copies,
+        },
+    });
+    try builder.setTerminator(entry, .end_thread);
+
+    const stored = program.instructions.get(copy_id).?.operation.parallel_copy;
+    try std.testing.expect(stored.register_copies.ptr != register_copies[0..].ptr);
+    try std.testing.expect(stored.flag_copies.ptr != flag_copies[0..].ptr);
+
+    register_copies[0].source.register = .{ .immediate = .{ .u32 = 42 } };
+    flag_copies[0].source = .{ .constant = false };
+    try std.testing.expect(stored.register_copies[0].source.register == .virtual);
+    try std.testing.expect(stored.flag_copies[0].source == .dynamic);
+
+    try validator.validate(&program);
+
+    const text = try printer.allocPrint(std.testing.allocator, &program);
+    defer std.testing.allocator.free(text);
+
+    try std.testing.expect(std.mem.indexOf(
+        u8,
+        text,
+        "parallel_copy [%destination:u32 <- %source:u32, %destination_flag <- (+%source_flag)]",
+    ) != null);
+
+    program.properties.parallel_copies_lowered = true;
+    try std.testing.expectError(error.UnloweredParallelCopy, validator.validate(&program));
+}
+
+test "[ir] pseudo: validator rejects invalid parallel copies" {
+    const std = @import("std");
+    const Builder = @import("Builder.zig");
+    const device = @import("../device.zig");
+    const program_ir = @import("program.zig");
+    const validator = @import("validator.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const register_id = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const flag_id = try builder.addVirtualFlag(.{});
+    const entry = try builder.addBlock("entry");
+    const instruction_id = try builder.appendInstruction(entry, .simd8, null, .{
+        .parallel_copy = .{
+            .register_copies = &.{},
+            .flag_copies = &.{},
+        },
+    });
+    try builder.setTerminator(entry, .end_thread);
+
+    try std.testing.expectError(error.EmptyParallelCopy, validator.validate(&program));
+
+    const inst = program.instructions.getMut(instruction_id).?;
+    inst.predicate = .{ .flag = .{ .virtual = flag_id } };
+    try std.testing.expectError(error.PredicatedParallelCopy, validator.validate(&program));
+    inst.predicate = null;
+
+    const duplicate_copy: RegisterCopy = .{
+        .destination = .{
+            .register = .{ .virtual = register_id },
+            .type = .u32,
+        },
+        .source = .{
+            .register = .{ .immediate = .{ .u32 = 1 } },
+            .type = .u32,
+            .region = operand.Region.broadcast(),
+        },
+    };
+    inst.operation = .{
+        .parallel_copy = .{
+            .register_copies = &.{ duplicate_copy, duplicate_copy },
+            .flag_copies = &.{},
+        },
+    };
+    try std.testing.expectError(error.DuplicateParallelCopyDestination, validator.validate(&program));
+}

src/intel/compiler/ir/validator.zig

@@ -0,0 +1,518 @@
+const ids = @import("id.zig");
+const instruction = @import("instruction.zig");
+const operand = @import("operand.zig");
+const program_ir = @import("program.zig");
+const pseudo = @import("pseudo.zig");
+
+pub const Error = error{
+    DuplicateBlockParameter,
+    DuplicateParallelCopyDestination,
+    EdgeArgumentCountMismatch,
+    EdgeArgumentKindMismatch,
+    EdgeArgumentTypeMismatch,
+    EmptyParallelCopy,
+    EntryBlockHasParameters,
+    InvalidBlock,
+    InvalidBufferAccess,
+    InvalidBufferReference,
+    InvalidDestination,
+    InvalidGlobalInvocationId,
+    InvalidImmediateType,
+    InvalidInstruction,
+    InvalidLaneCount,
+    InvalidMath,
+    InvalidMessage,
+    InvalidParallelCopyDestination,
+    InvalidPayloadLayout,
+    InvalidPhysicalRegister,
+    InvalidRegion,
+    InvalidRegisterAlignment,
+    InvalidRegisterSize,
+    InvalidStorageBuffer,
+    InvalidVirtualFlag,
+    InvalidVirtualRegister,
+    InvalidWorkgroupSize,
+    MissingEntryBlock,
+    MissingTerminator,
+    ParallelCopyTypeMismatch,
+    PredicatedParallelCopy,
+    UnallocatedVirtualFlag,
+    UnloweredBlockParameter,
+    UnloweredMessage,
+    UnloweredParallelCopy,
+    UnloweredResource,
+    UnloweredSystemValue,
+};
+
+pub fn validate(program: *const program_ir.Program) Error!void {
+    if (program.workgroup_size[0] == 0 or program.workgroup_size[1] == 0 or program.workgroup_size[2] == 0)
+        return Error.InvalidWorkgroupSize;
+
+    const entry_block = program.entry_block orelse return Error.MissingEntryBlock;
+    if (!program.blocks.isLive(entry_block))
+        return Error.InvalidBlock;
+
+    try validatePayload(program);
+
+    for (program.virtual_registers.entries.items) |entry| {
+        const register = entry orelse continue;
+        if (register.size_bytes == 0)
+            return Error.InvalidRegisterSize;
+        if (register.alignment_bytes == 0 or
+            (register.alignment_bytes & (register.alignment_bytes - 1)) != 0)
+            return Error.InvalidRegisterAlignment;
+        if (register.lane_count == 0)
+            return Error.InvalidLaneCount;
+    }
+
+    for (program.blocks.entries.items, 0..) |entry, block_index| {
+        const block = entry orelse continue;
+        if (block.terminator == null)
+            return Error.MissingTerminator;
+
+        const block_id = ids.BlockId.fromIndex(block_index);
+        if (block_id == entry_block and block.parameters.items.len != 0)
+            return Error.EntryBlockHasParameters;
+        if (program.properties.block_parameters_lowered and block.parameters.items.len != 0)
+            return Error.UnloweredBlockParameter;
+        for (block.parameters.items, 0..) |parameter, parameter_index|
+            try validateBlockParameter(program, block_index, parameter_index, parameter);
+
+        for (block.instructions.items) |instruction_id| {
+            const inst = program.instructions.get(instruction_id) orelse return Error.InvalidInstruction;
+            if (inst.parent_block != block_id)
+                return Error.InvalidInstruction;
+            try validateInstruction(program, inst.*);
+        }
+
+        try validateStructuredControl(program, block.structured_control);
+        try validateTerminator(program, block.terminator.?);
+    }
+}
+
+fn validatePayload(program: *const program_ir.Program) Error!void {
+    if (program.program_data.payload_grf_count > program.device_info.grf_count)
+        return Error.InvalidPayloadLayout;
+
+    if (program.payload.header_grf) |header| {
+        try validateRegisterRef(program, .{ .physical_grf = header });
+        if (header.byte_offset != 0)
+            return Error.InvalidPayloadLayout;
+    }
+}
+
+fn validateBlockParameter(program: *const program_ir.Program, block_index: usize, parameter_index: usize, parameter: pseudo.BlockParameter) Error!void {
+    switch (parameter) {
+        .register => |register_id| if (!program.virtual_registers.isLive(register_id))
+            return Error.InvalidVirtualRegister,
+        .flag => |flag_id| if (!program.virtual_flags.isLive(flag_id))
+            return Error.InvalidVirtualFlag,
+    }
+
+    for (program.blocks.entries.items, 0..) |entry, candidate_block_index| {
+        const block = entry orelse continue;
+        if (candidate_block_index > block_index)
+            break;
+        const limit = if (candidate_block_index == block_index) parameter_index else block.parameters.items.len;
+        for (block.parameters.items[0..limit]) |candidate| {
+            if (blockParametersEqual(parameter, candidate))
+                return Error.DuplicateBlockParameter;
+        }
+    }
+}
+
+fn blockParametersEqual(a: pseudo.BlockParameter, b: pseudo.BlockParameter) bool {
+    return switch (a) {
+        .register => |register_id| b == .register and b.register == register_id,
+        .flag => |flag_id| b == .flag and b.flag == flag_id,
+    };
+}
+
+fn validateInstruction(program: *const program_ir.Program, inst: instruction.Instruction) Error!void {
+    if (inst.predicate) |predicate|
+        try validateFlag(program, predicate.flag);
+
+    switch (inst.operation) {
+        .load_global_invocation_id => |op| {
+            if (program.properties.system_values_lowered)
+                return Error.UnloweredSystemValue;
+            try validateDestination(program, op.destination);
+            if (op.component >= 3 or op.destination.type != .u32)
+                return Error.InvalidGlobalInvocationId;
+        },
+        .load_buffer => |op| {
+            if (program.properties.messages_lowered)
+                return Error.UnloweredMessage;
+            try validateBufferReference(program, op.buffer);
+            try validateDestination(program, op.destination);
+            try validateBufferOffset(program, op.byte_offset);
+            if (!op.destination.type.isInitialTargetType())
+                return Error.InvalidBufferAccess;
+        },
+        .store_buffer => |op| {
+            if (program.properties.messages_lowered)
+                return Error.UnloweredMessage;
+            try validateBufferReference(program, op.buffer);
+            try validateBufferOffset(program, op.byte_offset);
+            try validateSource(program, op.source);
+            if (!op.source.type.isInitialTargetType())
+                return Error.InvalidBufferAccess;
+        },
+        .array_length => |op| {
+            try validateBufferReference(program, op.buffer);
+            try validateDestination(program, op.destination);
+            try validateBufferOffset(program, op.byte_offset);
+            if (op.destination.type != .u32 or op.stride == 0)
+                return Error.InvalidBufferAccess;
+        },
+        .surface_read => |op| {
+            try validateDestination(program, op.destination);
+            try validateBufferOffset(program, op.address);
+            if (!op.destination.type.isInitialTargetType())
+                return Error.InvalidBufferAccess;
+        },
+        .surface_write => |op| {
+            try validateBufferOffset(program, op.address);
+            try validateSource(program, op.data);
+            if (!op.data.type.isInitialTargetType())
+                return Error.InvalidBufferAccess;
+        },
+        .surface_message => |op| {
+            try validateRegisterSpan(program, op.payload);
+            if (!op.data_type.isInitialTargetType())
+                return Error.InvalidMessage;
+            switch (op.kind) {
+                .read => {
+                    if (op.payload.register_count != 1 or op.response == null)
+                        return Error.InvalidMessage;
+                    try validateRegisterSpan(program, op.response.?);
+                    if (op.response.?.register_count != 1)
+                        return Error.InvalidMessage;
+                },
+                .write => if (op.payload.register_count != 2 or op.response != null)
+                    return Error.InvalidMessage,
+            }
+        },
+        .move => |op| {
+            try validateDestination(program, op.destination);
+            try validateSource(program, op.source);
+        },
+        .binary => |op| {
+            try validateDestination(program, op.destination);
+            try validateSource(program, op.lhs);
+            try validateSource(program, op.rhs);
+        },
+        .compare => |op| {
+            try validateFlag(program, op.destination);
+            try validateSource(program, op.lhs);
+            try validateSource(program, op.rhs);
+        },
+        .math => |op| {
+            try validateDestination(program, op.destination);
+            try validateSource(program, op.lhs);
+            try validateSource(program, op.rhs);
+
+            switch (op.opcode) {
+                .integer_quotient => {
+                    if (inst.execution_size != .simd8)
+                        return Error.InvalidMath;
+                    if (op.destination.type != .u32 and op.destination.type != .i32)
+                        return Error.InvalidMath;
+                    if (op.lhs.type != op.destination.type or op.rhs.type != op.destination.type)
+                        return Error.InvalidMath;
+                },
+            }
+        },
+
+        .parallel_copy => |op| {
+            if (program.properties.parallel_copies_lowered)
+                return Error.UnloweredParallelCopy;
+            if (inst.predicate != null)
+                return Error.PredicatedParallelCopy;
+            try validateParallelCopy(program, op);
+        },
+    }
+}
+
+fn validateBufferReference(program: *const program_ir.Program, reference: instruction.BufferReference) Error!void {
+    switch (reference) {
+        .logical => |buffer| {
+            if (program.properties.resources_lowered)
+                return Error.UnloweredResource;
+            if (!program.storage_buffers.isLive(buffer))
+                return Error.InvalidStorageBuffer;
+        },
+        .binding_table => if (!program.properties.resources_lowered)
+            return Error.InvalidBufferReference,
+    }
+}
+
+fn validateBufferOffset(program: *const program_ir.Program, source: operand.Source) Error!void {
+    try validateSource(program, source);
+    if (source.type != .u32)
+        return Error.InvalidBufferAccess;
+}
+
+fn validateParallelCopy(program: *const program_ir.Program, copy: pseudo.ParallelCopy) Error!void {
+    if (copy.register_copies.len == 0 and copy.flag_copies.len == 0)
+        return Error.EmptyParallelCopy;
+
+    for (copy.register_copies, 0..) |item, index| {
+        try validateDestination(program, item.destination);
+        try validateSource(program, item.source);
+
+        if (item.destination.type != item.source.type)
+            return Error.ParallelCopyTypeMismatch;
+        if (item.destination.region.byte_offset != 0 or item.destination.region.horizontal_stride != 1)
+            return Error.InvalidParallelCopyDestination;
+
+        const destination_id = switch (item.destination.register) {
+            .virtual => |register_id| register_id,
+            else => return Error.InvalidParallelCopyDestination,
+        };
+        const destination_register = program.virtual_registers.get(destination_id) orelse
+            return Error.InvalidVirtualRegister;
+        if (destination_register.element_type != item.destination.type)
+            return Error.ParallelCopyTypeMismatch;
+
+        switch (item.source.register) {
+            .virtual => |source_id| {
+                const source_register = program.virtual_registers.get(source_id) orelse
+                    return Error.InvalidVirtualRegister;
+                if (source_register.element_type != item.source.type)
+                    return Error.ParallelCopyTypeMismatch;
+                if (!isBroadcast(item.source.region) and
+                    (source_register.size_bytes != destination_register.size_bytes or
+                        source_register.lane_count != destination_register.lane_count))
+                    return Error.ParallelCopyTypeMismatch;
+            },
+            .null => return Error.ParallelCopyTypeMismatch,
+            else => {},
+        }
+
+        for (copy.register_copies[0..index]) |previous| {
+            const previous_id = switch (previous.destination.register) {
+                .virtual => |register_id| register_id,
+                else => unreachable,
+            };
+            if (previous_id == destination_id)
+                return Error.DuplicateParallelCopyDestination;
+        }
+    }
+
+    for (copy.flag_copies, 0..) |item, index| {
+        if (!program.virtual_flags.isLive(item.destination))
+            return Error.InvalidVirtualFlag;
+        switch (item.source) {
+            .constant => {},
+            .dynamic => |predicate| try validateFlag(program, predicate.flag),
+        }
+
+        for (copy.flag_copies[0..index]) |previous| {
+            if (previous.destination == item.destination)
+                return Error.DuplicateParallelCopyDestination;
+        }
+    }
+}
+
+fn isBroadcast(region: operand.Region) bool {
+    return region.vertical_stride == 0 and region.width == 1 and region.horizontal_stride == 0;
+}
+
+fn validateSource(program: *const program_ir.Program, source: operand.Source) Error!void {
+    if (source.region.width == 0)
+        return Error.InvalidRegion;
+    try validateRegisterRef(program, source.register);
+
+    if (source.register == .immediate) {
+        const matches = switch (source.register.immediate) {
+            .u32 => source.type == .u32,
+            .i32 => source.type == .i32,
+            .f32 => source.type == .f32,
+        };
+        if (!matches)
+            return Error.InvalidImmediateType;
+    }
+}
+
+fn validateDestination(program: *const program_ir.Program, destination: operand.Destination) Error!void {
+    if (destination.region.horizontal_stride == 0)
+        return Error.InvalidRegion;
+    switch (destination.register) {
+        .immediate, .null => return Error.InvalidDestination,
+        else => try validateRegisterRef(program, destination.register),
+    }
+}
+
+fn validateRegisterSpan(program: *const program_ir.Program, span: operand.RegisterSpan) Error!void {
+    if (span.register_count == 0)
+        return Error.InvalidMessage;
+    switch (span.base) {
+        .virtual, .physical_grf => try validateRegisterRef(program, span.base),
+        else => return Error.InvalidMessage,
+    }
+}
+
+fn validateRegisterRef(program: *const program_ir.Program, register: operand.RegisterRef) Error!void {
+    switch (register) {
+        .virtual => |id| if (!program.virtual_registers.isLive(id))
+            return Error.InvalidVirtualRegister,
+        .physical_grf => |physical| {
+            if (physical.number >= program.device_info.grf_count or
+                physical.byte_offset >= program.device_info.grf_size_bytes)
+                return Error.InvalidPhysicalRegister;
+        },
+        .architecture, .immediate, .null => {},
+    }
+}
+
+fn validateFlag(program: *const program_ir.Program, flag: operand.FlagRef) Error!void {
+    switch (flag) {
+        .virtual => |id| {
+            if (program.properties.flags_allocated)
+                return Error.UnallocatedVirtualFlag;
+            if (!program.virtual_flags.isLive(id))
+                return Error.InvalidVirtualFlag;
+        },
+        .physical => {},
+    }
+}
+
+fn validateTerminator(program: *const program_ir.Program, terminator: instruction.Terminator) Error!void {
+    switch (terminator) {
+        .jump => |edge| try validateEdge(program, edge),
+        .conditional_branch => |branch| {
+            try validateFlag(program, branch.predicate.flag);
+            try validateEdge(program, branch.true_edge);
+            try validateEdge(program, branch.false_edge);
+        },
+        .end_thread, .@"unreachable" => {},
+    }
+}
+
+fn validateEdge(program: *const program_ir.Program, edge: instruction.Edge) Error!void {
+    try validateBlockTarget(program, edge.target);
+    const target = program.blocks.get(edge.target).?;
+
+    if (program.properties.block_parameters_lowered and edge.arguments.len != 0)
+        return Error.UnloweredBlockParameter;
+    if (edge.arguments.len != target.parameters.items.len)
+        return Error.EdgeArgumentCountMismatch;
+
+    for (target.parameters.items, edge.arguments) |parameter, argument| {
+        switch (parameter) {
+            .register => |destination_id| switch (argument) {
+                .source => |source| try validateRegisterEdgeArgument(program, destination_id, source),
+                .predicate => return Error.EdgeArgumentKindMismatch,
+            },
+            .flag => switch (argument) {
+                .source => return Error.EdgeArgumentKindMismatch,
+                .predicate => |predicate_value| switch (predicate_value) {
+                    .constant => {},
+                    .dynamic => |predicate| try validateFlag(program, predicate.flag),
+                },
+            },
+        }
+    }
+}
+
+fn validateRegisterEdgeArgument(program: *const program_ir.Program, destination_id: ids.VirtualRegisterId, source: operand.Source) Error!void {
+    const destination = program.virtual_registers.get(destination_id) orelse
+        return Error.InvalidVirtualRegister;
+    try validateSource(program, source);
+    if (source.type != destination.element_type)
+        return Error.EdgeArgumentTypeMismatch;
+
+    switch (source.register) {
+        .virtual => |source_id| {
+            const source_register = program.virtual_registers.get(source_id) orelse
+                return Error.InvalidVirtualRegister;
+            if (source_register.element_type != source.type)
+                return Error.EdgeArgumentTypeMismatch;
+            if (!isBroadcast(source.region) and
+                (source_register.size_bytes != destination.size_bytes or
+                    source_register.lane_count != destination.lane_count))
+                return Error.EdgeArgumentTypeMismatch;
+        },
+        .null => return Error.EdgeArgumentTypeMismatch,
+        else => {},
+    }
+}
+
+fn validateStructuredControl(program: *const program_ir.Program, control: instruction.StructuredControl) Error!void {
+    switch (control) {
+        .none => {},
+        .selection => |selection| try validateBlockTarget(program, selection.merge_block),
+        .loop => |loop| {
+            try validateBlockTarget(program, loop.merge_block);
+            try validateBlockTarget(program, loop.continue_block);
+        },
+    }
+}
+
+fn validateBlockTarget(program: *const program_ir.Program, block_id: ids.BlockId) Error!void {
+    if (!program.blocks.isLive(block_id))
+        return Error.InvalidBlock;
+}
+
+test "[ir] validator checks compute system values and resources" {
+    const std = @import("std");
+    const Builder = @import("Builder.zig");
+    const device = @import("../device.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const register = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const buffer = try builder.addStorageBuffer(.{ .set = 0, .binding = 0 });
+    const entry = try builder.addBlock("entry");
+    const system_value_id = try builder.appendInstruction(entry, .simd8, null, .{
+        .load_global_invocation_id = .{
+            .destination = .{ .register = .{ .virtual = register }, .type = .u32 },
+            .component = 0,
+        },
+    });
+    const buffer_load_id = try builder.appendInstruction(entry, .simd8, null, .{
+        .load_buffer = .{
+            .destination = .{ .register = .{ .virtual = register }, .type = .u32 },
+            .buffer = .{ .logical = buffer },
+            .byte_offset = .{
+                .register = .{ .immediate = .{ .u32 = 0 } },
+                .type = .u32,
+                .region = operand.Region.broadcast(),
+            },
+        },
+    });
+    try builder.setTerminator(entry, .end_thread);
+    try validate(&program);
+
+    program.instructions.getMut(system_value_id).?.operation.load_global_invocation_id.component = 3;
+    try std.testing.expectError(Error.InvalidGlobalInvocationId, validate(&program));
+    program.instructions.getMut(system_value_id).?.operation.load_global_invocation_id.component = 0;
+
+    program.properties.system_values_lowered = true;
+    try std.testing.expectError(Error.UnloweredSystemValue, validate(&program));
+    program.properties.system_values_lowered = false;
+
+    program.instructions.getMut(buffer_load_id).?.operation.load_buffer.buffer = .{ .logical = ids.StorageBufferId.fromIndex(99) };
+    try std.testing.expectError(Error.InvalidStorageBuffer, validate(&program));
+    program.instructions.getMut(buffer_load_id).?.operation.load_buffer.buffer = .{ .logical = buffer };
+
+    program.properties.resources_lowered = true;
+    try std.testing.expectError(Error.UnloweredResource, validate(&program));
+    program.instructions.getMut(buffer_load_id).?.operation.load_buffer.buffer = .{ .binding_table = 0 };
+    try validate(&program);
+}

src/intel/compiler/lower/block_arguments.zig

@@ -0,0 +1,272 @@
+const std = @import("std");
+const Builder = @import("../ir/Builder.zig");
+const ids = @import("../ir/id.zig");
+const instruction = @import("../ir/instruction.zig");
+const operand = @import("../ir/operand.zig");
+const program_ir = @import("../ir/program.zig");
+const pseudo = @import("../ir/pseudo.zig");
+const validator = @import("../ir/validator.zig");
+const device = @import("../device.zig");
+
+pub const Error = std.mem.Allocator.Error || error{
+    InvalidProgram,
+};
+
+pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!void {
+    validator.validate(program) catch return Error.InvalidProgram;
+    if (program.properties.block_parameters_lowered)
+        return;
+
+    var builder = Builder.init(program);
+    var original_blocks: std.ArrayList(ids.BlockId) = .empty;
+    defer original_blocks.deinit(allocator);
+
+    for (program.blocks.entries.items, 0..) |entry, index| {
+        _ = entry orelse continue;
+        try original_blocks.append(allocator, ids.BlockId.fromIndex(index));
+    }
+
+    var emitted_parallel_copy = false;
+    for (original_blocks.items) |block_id| {
+        const block = program.blocks.get(block_id) orelse return Error.InvalidProgram;
+        const terminator = block.terminator orelse return Error.InvalidProgram;
+        const rewritten: instruction.Terminator = switch (terminator) {
+            .jump => |edge| .{ .jump = try rewriteEdge(
+                allocator,
+                &builder,
+                edge,
+                &emitted_parallel_copy,
+            ) },
+            .conditional_branch => |branch| .{ .conditional_branch = .{
+                .predicate = branch.predicate,
+                .true_edge = try rewriteEdge(
+                    allocator,
+                    &builder,
+                    branch.true_edge,
+                    &emitted_parallel_copy,
+                ),
+                .false_edge = try rewriteEdge(
+                    allocator,
+                    &builder,
+                    branch.false_edge,
+                    &emitted_parallel_copy,
+                ),
+            } },
+            else => terminator,
+        };
+        builder.replaceTerminator(block_id, rewritten) catch |err| return mapBuilderError(err);
+    }
+
+    for (original_blocks.items) |block_id|
+        builder.clearBlockParameters(block_id) catch |err| return mapBuilderError(err);
+
+    program.properties.block_parameters_lowered = true;
+    if (emitted_parallel_copy)
+        program.properties.parallel_copies_lowered = false;
+}
+
+fn rewriteEdge(
+    allocator: std.mem.Allocator,
+    builder: *Builder,
+    edge: instruction.Edge,
+    emitted_parallel_copy: *bool,
+) Error!instruction.Edge {
+    if (edge.arguments.len == 0)
+        return .{ .target = edge.target, .arguments = &.{} };
+
+    const target = builder.program.blocks.get(edge.target) orelse return Error.InvalidProgram;
+    if (target.parameters.items.len != edge.arguments.len)
+        return Error.InvalidProgram;
+
+    var register_copies: std.ArrayList(pseudo.RegisterCopy) = .empty;
+    defer register_copies.deinit(allocator);
+    var flag_copies: std.ArrayList(pseudo.FlagCopy) = .empty;
+    defer flag_copies.deinit(allocator);
+
+    for (target.parameters.items, edge.arguments) |parameter, argument| {
+        switch (parameter) {
+            .register => |destination_id| {
+                const source = switch (argument) {
+                    .source => |value| value,
+                    .predicate => return Error.InvalidProgram,
+                };
+                const destination = builder.program.virtual_registers.get(destination_id) orelse
+                    return Error.InvalidProgram;
+                try register_copies.append(allocator, .{
+                    .destination = .{
+                        .register = .{ .virtual = destination_id },
+                        .type = destination.element_type,
+                    },
+                    .source = source,
+                });
+            },
+            .flag => |destination_id| {
+                const source = switch (argument) {
+                    .source => return Error.InvalidProgram,
+                    .predicate => |value| value,
+                };
+                try flag_copies.append(allocator, .{
+                    .destination = destination_id,
+                    .source = source,
+                });
+            },
+        }
+    }
+
+    const edge_block = builder.addBlock(null) catch |err| return mapBuilderError(err);
+    _ = builder.appendInstruction(edge_block, executionSize(builder.program.dispatch_width), null, .{
+        .parallel_copy = .{
+            .register_copies = register_copies.items,
+            .flag_copies = flag_copies.items,
+        },
+    }) catch |err| return mapBuilderError(err);
+    builder.setTerminator(edge_block, .{ .jump = .{
+        .target = edge.target,
+        .arguments = &.{},
+    } }) catch |err| return mapBuilderError(err);
+
+    emitted_parallel_copy.* = true;
+    return .{ .target = edge_block, .arguments = &.{} };
+}
+
+fn executionSize(dispatch_width: device.DispatchWidth) device.ExecutionSize {
+    return @enumFromInt(@intFromEnum(dispatch_width));
+}
+
+fn mapBuilderError(err: anyerror) Error {
+    return switch (err) {
+        Error.OutOfMemory => Error.OutOfMemory,
+        else => Error.InvalidProgram,
+    };
+}
+
+test "[ir] block arguments: lower register and flag parameters" {
+    const printer = @import("../ir/printer.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const source_register = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const destination_register = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const source_flag = try builder.addVirtualFlag(.{});
+    const destination_flag = try builder.addVirtualFlag(.{});
+    const entry = try builder.addBlock("entry");
+    const merge = try builder.addBlock("merge");
+
+    try builder.addBlockParameter(merge, .{ .register = destination_register });
+    try builder.addBlockParameter(merge, .{ .flag = destination_flag });
+    try builder.setTerminator(entry, .{ .jump = try builder.edge(merge, &.{
+        .{ .source = .{
+            .register = .{ .virtual = source_register },
+            .type = .u32,
+            .region = operand.Region.contiguous(.simd8),
+        } },
+        .{ .predicate = .{ .dynamic = .{ .flag = .{ .virtual = source_flag } } } },
+    }) });
+    try builder.setTerminator(merge, .end_thread);
+
+    try validator.validate(&program);
+    const before = try printer.allocPrint(std.testing.allocator, &program);
+    defer std.testing.allocator.free(before);
+    try std.testing.expect(std.mem.indexOf(u8, before, ".merge(%v1, %f1):") != null);
+    try std.testing.expect(std.mem.indexOf(u8, before, "jump .merge(%v0:u32, (+%f0))") != null);
+
+    try run(std.testing.allocator, &program);
+    try validator.validate(&program);
+
+    try std.testing.expect(program.properties.block_parameters_lowered);
+    try std.testing.expect(!program.properties.parallel_copies_lowered);
+    try std.testing.expectEqual(@as(usize, 0), program.blocks.get(merge).?.parameters.items.len);
+
+    const edge_block_id = program.blocks.get(entry).?.terminator.?.jump.target;
+    try std.testing.expect(edge_block_id != merge);
+    try std.testing.expectEqual(@as(usize, 0), program.blocks.get(entry).?.terminator.?.jump.arguments.len);
+
+    const edge_block = program.blocks.get(edge_block_id).?;
+    try std.testing.expectEqual(@as(usize, 1), edge_block.instructions.items.len);
+    const copy = program.instructions.get(edge_block.instructions.items[0]).?.operation.parallel_copy;
+    try std.testing.expectEqual(@as(usize, 1), copy.register_copies.len);
+    try std.testing.expectEqual(destination_register, copy.register_copies[0].destination.register.virtual);
+    try std.testing.expectEqual(source_register, copy.register_copies[0].source.register.virtual);
+    try std.testing.expectEqual(@as(usize, 1), copy.flag_copies.len);
+    try std.testing.expectEqual(destination_flag, copy.flag_copies[0].destination);
+    try std.testing.expectEqual(source_flag, copy.flag_copies[0].source.dynamic.flag.virtual);
+    try std.testing.expectEqual(merge, edge_block.terminator.?.jump.target);
+}
+
+test "[ir] block arguments: split same-target conditional edges" {
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const destination = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const condition = try builder.addVirtualFlag(.{});
+    const entry = try builder.addBlock("entry");
+    const merge = try builder.addBlock("merge");
+    try builder.addBlockParameter(merge, .{ .register = destination });
+
+    const one: pseudo.EdgeArgument = .{ .source = .{
+        .register = .{ .immediate = .{ .u32 = 1 } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    } };
+    const two: pseudo.EdgeArgument = .{ .source = .{
+        .register = .{ .immediate = .{ .u32 = 2 } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    } };
+    try builder.setTerminator(entry, .{ .conditional_branch = .{
+        .predicate = .{ .flag = .{ .virtual = condition } },
+        .true_edge = try builder.edge(merge, &.{one}),
+        .false_edge = try builder.edge(merge, &.{two}),
+    } });
+    try builder.setTerminator(merge, .end_thread);
+
+    try run(std.testing.allocator, &program);
+    try validator.validate(&program);
+
+    const branch = program.blocks.get(entry).?.terminator.?.conditional_branch;
+    try std.testing.expect(branch.true_edge.target != branch.false_edge.target);
+    try std.testing.expect(branch.true_edge.target != merge);
+    try std.testing.expect(branch.false_edge.target != merge);
+
+    const true_block = program.blocks.get(branch.true_edge.target).?;
+    const false_block = program.blocks.get(branch.false_edge.target).?;
+    const true_copy = program.instructions.get(true_block.instructions.items[0]).?.operation.parallel_copy;
+    const false_copy = program.instructions.get(false_block.instructions.items[0]).?.operation.parallel_copy;
+    try std.testing.expectEqual(@as(u32, 1), true_copy.register_copies[0].source.register.immediate.u32);
+    try std.testing.expectEqual(@as(u32, 2), false_copy.register_copies[0].source.register.immediate.u32);
+}

src/intel/compiler/lower/lower.zig

@@ -0,0 +1,3 @@
+pub const block_arguments = @import("block_arguments.zig");
+pub const common_ir = @import("common_ir.zig");
+pub const parallel_copies = @import("parallel_copies.zig");

src/intel/compiler/lower/parallel_copies.zig

@@ -0,0 +1,466 @@
+const std = @import("std");
+const device = @import("../device.zig");
+const Builder = @import("../ir/Builder.zig");
+const ids = @import("../ir/id.zig");
+const instruction = @import("../ir/instruction.zig");
+const operand = @import("../ir/operand.zig");
+const program_ir = @import("../ir/program.zig");
+const pseudo = @import("../ir/pseudo.zig");
+const validator = @import("../ir/validator.zig");
+
+pub const Error = std.mem.Allocator.Error || error{
+    InvalidProgram,
+};
+
+const EmittedInstruction = struct {
+    predicate: ?operand.Predicate = null,
+    operation: instruction.Operation,
+};
+
+const FlagValue = union(enum) {
+    constant: bool,
+    snapshot: ids.VirtualRegisterId,
+};
+
+const FlagWrite = struct {
+    destination: ids.VirtualFlagId,
+    value: FlagValue,
+};
+
+pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!void {
+    validator.validate(program) catch return Error.InvalidProgram;
+    if (program.properties.parallel_copies_lowered)
+        return;
+
+    var builder = Builder.init(program);
+    for (program.blocks.entries.items, 0..) |entry, block_index| {
+        _ = entry orelse continue;
+        const block_id = ids.BlockId.fromIndex(block_index);
+        var instruction_index: usize = 0;
+
+        while (true) {
+            const block = program.blocks.get(block_id) orelse return Error.InvalidProgram;
+            if (instruction_index >= block.instructions.items.len)
+                break;
+
+            const instruction_id = block.instructions.items[instruction_index];
+            const inst = program.instructions.get(instruction_id) orelse return Error.InvalidProgram;
+            const parallel_copy = switch (inst.operation) {
+                .parallel_copy => |copy| copy,
+                else => {
+                    instruction_index += 1;
+                    continue;
+                },
+            };
+            if (inst.predicate != null)
+                return Error.InvalidProgram;
+            const execution_size = inst.execution_size;
+
+            var emitted: std.ArrayList(EmittedInstruction) = .empty;
+            defer emitted.deinit(allocator);
+            try lowerParallelCopy(allocator, &builder, execution_size, parallel_copy, &emitted);
+
+            if (emitted.items.len == 0) {
+                const mutable_block = program.blocks.getMut(block_id) orelse return Error.InvalidProgram;
+                const removed_id = mutable_block.instructions.orderedRemove(instruction_index);
+                if (removed_id != instruction_id or !program.instructions.remove(instruction_id))
+                    return Error.InvalidProgram;
+                continue;
+            }
+
+            builder.replaceOperation(instruction_id, emitted.items[0].operation) catch |err|
+                return mapBuilderError(err);
+            const replacement = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
+            replacement.predicate = emitted.items[0].predicate;
+
+            for (emitted.items[1..], 1..) |item, offset| {
+                _ = builder.insertInstruction(
+                    block_id,
+                    instruction_index + offset,
+                    execution_size,
+                    item.predicate,
+                    item.operation,
+                ) catch |err| return mapBuilderError(err);
+            }
+            instruction_index += emitted.items.len;
+        }
+    }
+
+    program.properties.parallel_copies_lowered = true;
+    validator.validate(program) catch return Error.InvalidProgram;
+}
+
+fn lowerParallelCopy(
+    allocator: std.mem.Allocator,
+    builder: *Builder,
+    execution_size: device.ExecutionSize,
+    copy: pseudo.ParallelCopy,
+    emitted: *std.ArrayList(EmittedInstruction),
+) Error!void {
+    var pending_registers: std.ArrayList(pseudo.RegisterCopy) = .empty;
+    defer pending_registers.deinit(allocator);
+    for (copy.register_copies) |item| {
+        if (!isRegisterIdentity(item, execution_size))
+            try pending_registers.append(allocator, item);
+    }
+
+    var flag_writes: std.ArrayList(FlagWrite) = .empty;
+    defer flag_writes.deinit(allocator);
+    try snapshotFlagSources(allocator, builder, execution_size, copy.flag_copies, emitted, &flag_writes);
+    try scheduleRegisterCopies(allocator, builder, execution_size, &pending_registers, emitted);
+    try emitFlagWrites(allocator, execution_size, flag_writes.items, emitted);
+}
+
+fn scheduleRegisterCopies(
+    allocator: std.mem.Allocator,
+    builder: *Builder,
+    execution_size: device.ExecutionSize,
+    pending: *std.ArrayList(pseudo.RegisterCopy),
+    emitted: *std.ArrayList(EmittedInstruction),
+) Error!void {
+    while (pending.items.len != 0) {
+        if (findReadyCopy(pending.items)) |ready_index| {
+            const ready = pending.orderedRemove(ready_index);
+            try emitted.append(allocator, .{ .operation = .{ .move = .{
+                .destination = ready.destination,
+                .source = ready.source,
+            } } });
+            continue;
+        }
+
+        const cycle_copy = &pending.items[0];
+        const destination_id = destinationVirtualRegister(cycle_copy.destination) orelse
+            return Error.InvalidProgram;
+        const destination_register = builder.program.virtual_registers.get(destination_id) orelse
+            return Error.InvalidProgram;
+        const temporary = builder.addVirtualRegister(.{
+            .size_bytes = destination_register.size_bytes,
+            .alignment_bytes = destination_register.alignment_bytes,
+            .element_type = destination_register.element_type,
+            .lane_count = destination_register.lane_count,
+            .class = .temporary,
+            .spillable = destination_register.spillable,
+        }) catch |err| return mapBuilderError(err);
+
+        var temporary_destination = cycle_copy.destination;
+        temporary_destination.register = .{ .virtual = temporary };
+        try emitted.append(allocator, .{ .operation = .{ .move = .{
+            .destination = temporary_destination,
+            .source = cycle_copy.source,
+        } } });
+
+        cycle_copy.source = .{
+            .register = .{ .virtual = temporary },
+            .type = cycle_copy.source.type,
+            .region = operand.Region.contiguous(execution_size),
+        };
+    }
+}
+
+fn findReadyCopy(pending: []const pseudo.RegisterCopy) ?usize {
+    for (pending, 0..) |candidate, candidate_index| {
+        const destination_id = destinationVirtualRegister(candidate.destination) orelse continue;
+        var destination_is_source = false;
+        for (pending, 0..) |other, other_index| {
+            if (candidate_index == other_index)
+                continue;
+            switch (other.source.register) {
+                .virtual => |source_id| if (source_id == destination_id) {
+                    destination_is_source = true;
+                    break;
+                },
+                else => {},
+            }
+        }
+        if (!destination_is_source)
+            return candidate_index;
+    }
+    return null;
+}
+
+fn snapshotFlagSources(
+    allocator: std.mem.Allocator,
+    builder: *Builder,
+    execution_size: device.ExecutionSize,
+    copies: []const pseudo.FlagCopy,
+    emitted: *std.ArrayList(EmittedInstruction),
+    writes: *std.ArrayList(FlagWrite),
+) Error!void {
+    for (copies) |copy| {
+        if (isFlagIdentity(copy))
+            continue;
+
+        const value: FlagValue = switch (copy.source) {
+            .constant => |constant| .{ .constant = constant },
+            .dynamic => |predicate| value: {
+                const temporary = builder.addVirtualRegister(.{
+                    .size_bytes = @as(u32, @intFromEnum(execution_size)) * @sizeOf(u32),
+                    .alignment_bytes = builder.program.device_info.grf_size_bytes,
+                    .element_type = .u32,
+                    .lane_count = @intFromEnum(execution_size),
+                    .class = .temporary,
+                }) catch |err| return mapBuilderError(err);
+                const destination: operand.Destination = .{
+                    .register = .{ .virtual = temporary },
+                    .type = .u32,
+                };
+                try emitted.append(allocator, .{ .operation = .{ .move = .{
+                    .destination = destination,
+                    .source = immediateU32(0),
+                } } });
+                try emitted.append(allocator, .{
+                    .predicate = predicate,
+                    .operation = .{ .move = .{
+                        .destination = destination,
+                        .source = immediateU32(1),
+                    } },
+                });
+                break :value .{ .snapshot = temporary };
+            },
+        };
+        try writes.append(allocator, .{
+            .destination = copy.destination,
+            .value = value,
+        });
+    }
+}
+
+fn emitFlagWrites(
+    allocator: std.mem.Allocator,
+    execution_size: device.ExecutionSize,
+    writes: []const FlagWrite,
+    emitted: *std.ArrayList(EmittedInstruction),
+) Error!void {
+    for (writes) |write| {
+        const value = switch (write.value) {
+            .constant => |constant| immediateU32(@intFromBool(constant)),
+            .snapshot => |temporary| operand.Source{
+                .register = .{ .virtual = temporary },
+                .type = .u32,
+                .region = operand.Region.contiguous(execution_size),
+            },
+        };
+        try emitted.append(allocator, .{ .operation = .{ .compare = .{
+            .opcode = .not_equal,
+            .destination = .{ .virtual = write.destination },
+            .lhs = value,
+            .rhs = immediateU32(0),
+        } } });
+    }
+}
+
+fn isRegisterIdentity(copy: pseudo.RegisterCopy, execution_size: device.ExecutionSize) bool {
+    const destination_id = destinationVirtualRegister(copy.destination) orelse return false;
+    const source_id = switch (copy.source.register) {
+        .virtual => |id| id,
+        else => return false,
+    };
+    if (destination_id != source_id or copy.source.negate or copy.source.absolute)
+        return false;
+
+    const contiguous = operand.Region.contiguous(execution_size);
+    return copy.destination.type == copy.source.type and
+        copy.destination.region.byte_offset == contiguous.byte_offset and
+        copy.destination.region.horizontal_stride == 1 and
+        copy.source.region.byte_offset == contiguous.byte_offset and
+        copy.source.region.vertical_stride == contiguous.vertical_stride and
+        copy.source.region.width == contiguous.width and
+        copy.source.region.horizontal_stride == contiguous.horizontal_stride;
+}
+
+fn isFlagIdentity(copy: pseudo.FlagCopy) bool {
+    return switch (copy.source) {
+        .constant => false,
+        .dynamic => |predicate| !predicate.inverse and switch (predicate.flag) {
+            .virtual => |source| source == copy.destination,
+            .physical => false,
+        },
+    };
+}
+
+fn destinationVirtualRegister(destination: operand.Destination) ?ids.VirtualRegisterId {
+    return switch (destination.register) {
+        .virtual => |id| id,
+        else => null,
+    };
+}
+
+fn immediateU32(value: u32) operand.Source {
+    return .{
+        .register = .{ .immediate = .{ .u32 = value } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    };
+}
+
+fn mapBuilderError(err: anyerror) Error {
+    return switch (err) {
+        Error.OutOfMemory => Error.OutOfMemory,
+        else => Error.InvalidProgram,
+    };
+}
+
+const test_device_info: device.DeviceInfo = .{
+    .generation = .gen9,
+    .platform = .skylake,
+    .pci_device_id = 0x1912,
+    .grf_count = 128,
+};
+
+fn addTestRegister(builder: *Builder) !ids.VirtualRegisterId {
+    return builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+}
+
+fn testDestination(register: ids.VirtualRegisterId) operand.Destination {
+    return .{
+        .register = .{ .virtual = register },
+        .type = .u32,
+    };
+}
+
+fn testSource(register: ids.VirtualRegisterId) operand.Source {
+    return .{
+        .register = .{ .virtual = register },
+        .type = .u32,
+        .region = operand.Region.contiguous(.simd8),
+    };
+}
+
+test "[intel] parallel copies: lower independent copies" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const source_a = try addTestRegister(&builder);
+    const source_b = try addTestRegister(&builder);
+    const destination_a = try addTestRegister(&builder);
+    const destination_b = try addTestRegister(&builder);
+    const entry = try builder.addBlock("entry");
+    const copies = [_]pseudo.RegisterCopy{
+        .{ .destination = testDestination(destination_a), .source = testSource(source_a) },
+        .{ .destination = testDestination(destination_b), .source = testSource(source_b) },
+    };
+    _ = try builder.appendInstruction(entry, .simd8, null, .{ .parallel_copy = .{
+        .register_copies = &copies,
+        .flag_copies = &.{},
+    } });
+    try builder.setTerminator(entry, .end_thread);
+
+    try validator.validate(&program);
+    try run(std.testing.allocator, &program);
+    try validator.validate(&program);
+
+    try std.testing.expect(program.properties.parallel_copies_lowered);
+    const instructions = program.blocks.get(entry).?.instructions.items;
+    try std.testing.expectEqual(@as(usize, 2), instructions.len);
+    const first = program.instructions.get(instructions[0]).?;
+    const second = program.instructions.get(instructions[1]).?;
+    try std.testing.expect(first.operation == .move);
+    try std.testing.expect(second.operation == .move);
+    try std.testing.expectEqual(destination_a, first.operation.move.destination.register.virtual);
+    try std.testing.expectEqual(source_a, first.operation.move.source.register.virtual);
+    try std.testing.expectEqual(destination_b, second.operation.move.destination.register.virtual);
+    try std.testing.expectEqual(source_b, second.operation.move.source.register.virtual);
+}
+
+test "[intel] parallel copies: remove identity copies" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const register = try addTestRegister(&builder);
+    const entry = try builder.addBlock("entry");
+    const copies = [_]pseudo.RegisterCopy{.{
+        .destination = testDestination(register),
+        .source = testSource(register),
+    }};
+    const copy_id = try builder.appendInstruction(entry, .simd8, null, .{ .parallel_copy = .{
+        .register_copies = &copies,
+        .flag_copies = &.{},
+    } });
+    try builder.setTerminator(entry, .end_thread);
+
+    try validator.validate(&program);
+    try run(std.testing.allocator, &program);
+    try validator.validate(&program);
+
+    try std.testing.expect(program.properties.parallel_copies_lowered);
+    try std.testing.expectEqual(@as(usize, 0), program.blocks.get(entry).?.instructions.items.len);
+    try std.testing.expect(program.instructions.get(copy_id) == null);
+}
+
+test "[intel] parallel copies: break a two-register cycle" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const register_a = try addTestRegister(&builder);
+    const register_b = try addTestRegister(&builder);
+    const entry = try builder.addBlock("entry");
+    const copies = [_]pseudo.RegisterCopy{
+        .{ .destination = testDestination(register_a), .source = testSource(register_b) },
+        .{ .destination = testDestination(register_b), .source = testSource(register_a) },
+    };
+    _ = try builder.appendInstruction(entry, .simd8, null, .{ .parallel_copy = .{
+        .register_copies = &copies,
+        .flag_copies = &.{},
+    } });
+    try builder.setTerminator(entry, .end_thread);
+
+    try validator.validate(&program);
+    try run(std.testing.allocator, &program);
+    try validator.validate(&program);
+
+    const instructions = program.blocks.get(entry).?.instructions.items;
+    try std.testing.expectEqual(@as(usize, 3), instructions.len);
+    const snapshot = program.instructions.get(instructions[0]).?.operation.move;
+    const restore_b = program.instructions.get(instructions[1]).?.operation.move;
+    const restore_a = program.instructions.get(instructions[2]).?.operation.move;
+    const temporary = snapshot.destination.register.virtual;
+
+    try std.testing.expect(temporary != register_a and temporary != register_b);
+    try std.testing.expectEqual(register_b, snapshot.source.register.virtual);
+    try std.testing.expectEqual(register_b, restore_b.destination.register.virtual);
+    try std.testing.expectEqual(register_a, restore_b.source.register.virtual);
+    try std.testing.expectEqual(register_a, restore_a.destination.register.virtual);
+    try std.testing.expectEqual(temporary, restore_a.source.register.virtual);
+    try std.testing.expectEqual(operand.RegisterClass.temporary, program.virtual_registers.get(temporary).?.class);
+}
+
+test "[intel] parallel copies: snapshot flag cycles" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const flag_a = try builder.addVirtualFlag(.{});
+    const flag_b = try builder.addVirtualFlag(.{});
+    const entry = try builder.addBlock("entry");
+    const copies = [_]pseudo.FlagCopy{
+        .{ .destination = flag_a, .source = .{ .dynamic = .{ .flag = .{ .virtual = flag_b } } } },
+        .{ .destination = flag_b, .source = .{ .dynamic = .{ .flag = .{ .virtual = flag_a } } } },
+    };
+    _ = try builder.appendInstruction(entry, .simd8, null, .{ .parallel_copy = .{
+        .register_copies = &.{},
+        .flag_copies = &copies,
+    } });
+    try builder.setTerminator(entry, .end_thread);
+
+    try validator.validate(&program);
+    try run(std.testing.allocator, &program);
+    try validator.validate(&program);
+
+    const instructions = program.blocks.get(entry).?.instructions.items;
+    try std.testing.expectEqual(@as(usize, 6), instructions.len);
+    for (instructions[0..4]) |instruction_id|
+        try std.testing.expect(program.instructions.get(instruction_id).?.operation == .move);
+    try std.testing.expect(program.instructions.get(instructions[4]).?.operation == .compare);
+    try std.testing.expect(program.instructions.get(instructions[5]).?.operation == .compare);
+    try std.testing.expectEqual(flag_a, program.instructions.get(instructions[4]).?.operation.compare.destination.virtual);
+    try std.testing.expectEqual(flag_b, program.instructions.get(instructions[5]).?.operation.compare.destination.virtual);
+}

src/intel/compiler/targets/gen9/compute/abi.zig

@@ -0,0 +1,60 @@
+const operand = @import("../../../ir/operand.zig");
+const program_ir = @import("../../../ir/program.zig");
+
+pub const Error = error{
+    InvalidPayloadLayout,
+};
+
+const thread_header: operand.PhysicalGrf = .{
+    .number = 0,
+    .byte_offset = 0,
+};
+
+pub fn run(program: *program_ir.Program) Error!void {
+    if (program.properties.compute_abi_lowered)
+        return;
+
+    if (program.payload.header_grf) |header| {
+        if (header.number != thread_header.number or header.byte_offset != thread_header.byte_offset)
+            return Error.InvalidPayloadLayout;
+    }
+    if (program.program_data.payload_grf_count > 1)
+        return Error.InvalidPayloadLayout;
+
+    program.payload.header_grf = thread_header;
+    program.program_data.payload_grf_count = 1;
+    program.properties.compute_abi_lowered = true;
+}
+
+const std = @import("std");
+const device = @import("../../../device.zig");
+
+const test_device: device.DeviceInfo = .{
+    .generation = .gen9,
+    .platform = .skylake,
+    .pci_device_id = 0x1912,
+    .grf_count = 128,
+};
+
+test "[gen9] compute ABI: reserve thread header" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    try run(&program);
+
+    try std.testing.expectEqual(thread_header, program.payload.header_grf.?);
+    try std.testing.expectEqual(@as(u16, 1), program.program_data.payload_grf_count);
+    try std.testing.expect(program.properties.compute_abi_lowered);
+
+    try run(&program);
+    try std.testing.expectEqual(@as(u16, 1), program.program_data.payload_grf_count);
+}
+
+test "[gen9] compute ABI: reject conflicting payload" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+    program.payload.header_grf = .{ .number = 1 };
+
+    try std.testing.expectError(Error.InvalidPayloadLayout, run(&program));
+    try std.testing.expect(!program.properties.compute_abi_lowered);
+}

src/intel/compiler/targets/gen9/compute/array_length_lowering.zig

@@ -0,0 +1,91 @@
+const std = @import("std");
+
+const Builder = @import("../../../ir/Builder.zig");
+const ids = @import("../../../ir/id.zig");
+const operand = @import("../../../ir/operand.zig");
+const program_ir = @import("../../../ir/program.zig");
+const resource_layout = @import("resource_layout.zig");
+
+pub const Error = std.mem.Allocator.Error || error{InvalidProgram};
+
+pub fn run(program: *program_ir.Program, layout: *const resource_layout.Layout) Error!void {
+    if (!program.properties.resources_lowered)
+        return Error.InvalidProgram;
+    if (layout.bindings.len >= std.math.maxInt(u8))
+        return Error.InvalidProgram;
+
+    var builder = Builder.init(program);
+    for (program.blocks.entries.items, 0..) |entry, block_index| {
+        _ = entry orelse continue;
+        const block_id = ids.BlockId.fromIndex(block_index);
+        var instruction_index: usize = 0;
+
+        while (true) {
+            const block = program.blocks.get(block_id) orelse return Error.InvalidProgram;
+            if (instruction_index >= block.instructions.items.len)
+                break;
+
+            const instruction_id = block.instructions.items[instruction_index];
+            const inst = program.instructions.get(instruction_id) orelse return Error.InvalidProgram;
+            const op = switch (inst.operation) {
+                .array_length => |value| value,
+                else => {
+                    instruction_index += 1;
+                    continue;
+                },
+            };
+            const resource_index = switch (op.buffer) {
+                .binding_table => |value| value,
+                .logical => return Error.InvalidProgram,
+            };
+            if (resource_index >= layout.bindings.len or op.stride == 0)
+                return Error.InvalidProgram;
+
+            const execution_size = inst.execution_size;
+            const predicate = inst.predicate;
+            const result_source: operand.Source = .{
+                .register = op.destination.register,
+                .type = .u32,
+                .region = operand.Region.contiguous(execution_size),
+            };
+            var negated_offset = op.byte_offset;
+            negated_offset.negate = !negated_offset.negate;
+
+            const mutable = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
+            mutable.operation = .{ .load_buffer = .{
+                .destination = op.destination,
+                .buffer = .{ .binding_table = @intCast(layout.bindings.len) },
+                .byte_offset = immediate(@as(u32, resource_index) * @sizeOf(u32)),
+            } };
+
+            _ = builder.insertInstruction(block_id, instruction_index + 1, execution_size, predicate, .{ .binary = .{
+                .opcode = .add,
+                .destination = op.destination,
+                .lhs = result_source,
+                .rhs = negated_offset,
+            } }) catch |err| return mapBuilderError(err);
+            _ = builder.insertInstruction(block_id, instruction_index + 2, execution_size, predicate, .{ .math = .{
+                .opcode = .integer_quotient,
+                .destination = op.destination,
+                .lhs = result_source,
+                .rhs = immediate(op.stride),
+            } }) catch |err| return mapBuilderError(err);
+            instruction_index += 3;
+        }
+    }
+}
+
+fn immediate(value: u32) operand.Source {
+    return .{
+        .register = .{ .immediate = .{ .u32 = value } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    };
+}
+
+fn mapBuilderError(err: Builder.Error) Error {
+    return switch (err) {
+        error.OutOfMemory => Error.OutOfMemory,
+        else => Error.InvalidProgram,
+    };
+}

src/intel/compiler/targets/gen9/compute/compute.zig

@@ -0,0 +1,34 @@
+const std = @import("std");
+
+pub const abi = @import("abi.zig");
+pub const dispatch = @import("dispatch.zig");
+pub const eu_encoder = @import("eu_encoder.zig");
+pub const kernel_encoder = @import("kernel_encoder.zig");
+pub const message_addresses = @import("message_addresses.zig");
+pub const message_descriptor = @import("message_descriptor.zig");
+pub const message_lowering = @import("message_lowering.zig");
+pub const message_payloads = @import("message_payloads.zig");
+pub const resource_layout = @import("resource_layout.zig");
+pub const resource_lowering = @import("resource_lowering.zig");
+pub const regions = @import("regions.zig");
+pub const system_values = @import("system_values.zig");
+pub const ResourceLayout = resource_layout.Layout;
+
+pub const Error = error{UnsupportedWorkgroupSize};
+
+pub fn validateWorkgroupSize(size: [3]u32) Error!void {
+    if (size[0] == 0 or size[1] == 0 or size[2] == 0 or size[0] > 128 or size[1] > 128 or size[2] > 64)
+        return Error.UnsupportedWorkgroupSize;
+    const xy = std.math.mul(u32, size[0], size[1]) catch return Error.UnsupportedWorkgroupSize;
+    const invocations = std.math.mul(u32, xy, size[2]) catch return Error.UnsupportedWorkgroupSize;
+    if (invocations > 128)
+        return Error.UnsupportedWorkgroupSize;
+}
+
+test "[gen9] compute: validate workgroup limits" {
+    try validateWorkgroupSize(.{ 1, 1, 1 });
+    try validateWorkgroupSize(.{ 128, 1, 1 });
+    try std.testing.expectError(Error.UnsupportedWorkgroupSize, validateWorkgroupSize(.{ 0, 1, 1 }));
+    try std.testing.expectError(Error.UnsupportedWorkgroupSize, validateWorkgroupSize(.{ 129, 1, 1 }));
+    try std.testing.expectError(Error.UnsupportedWorkgroupSize, validateWorkgroupSize(.{ 64, 3, 1 }));
+}

src/intel/compiler/targets/gen9/compute/dispatch.zig

@@ -0,0 +1,209 @@
+const std = @import("std");
+
+pub const max_storage_surfaces: usize = 4;
+pub const max_surfaces: usize = max_storage_surfaces + 1;
+pub const page_size: usize = 4096;
+pub const surface_state_size: usize = 64;
+pub const interface_descriptor_size: usize = 32;
+
+const mocs: u32 = 0x78;
+pub const base_address_delta: u32 = 1 | (mocs << 4);
+const raw_surface_format: u32 = 0x1ff;
+
+pub const Error = error{
+    EmptyBuffer,
+    StateTooLarge,
+    UnsupportedBufferSize,
+    TooManySurfaces,
+};
+
+pub const StateLayout = struct {
+    size: usize,
+    kernel_offset: u32,
+    surface_offsets: [max_surfaces]u32,
+    surface_address_offsets: [max_surfaces]u32,
+    surface_count: u8,
+    storage_surface_count: u8,
+    size_table_offset: u32,
+    binding_table_offset: u32,
+    interface_descriptor_offset: u32,
+};
+
+pub fn writeState(destination: []u8, kernel: []const u8, buffer_sizes: []const u64) Error!StateLayout {
+    if (buffer_sizes.len > max_storage_surfaces)
+        return Error.TooManySurfaces;
+
+    var layout: StateLayout = .{
+        .size = 0,
+        .kernel_offset = 0,
+        .surface_offsets = @splat(0),
+        .surface_address_offsets = @splat(0),
+        .surface_count = @intCast(buffer_sizes.len + 1),
+        .storage_surface_count = @intCast(buffer_sizes.len),
+        .size_table_offset = 0,
+        .binding_table_offset = 0,
+        .interface_descriptor_offset = 0,
+    };
+
+    var cursor = alignForward(kernel.len, 64);
+    for (buffer_sizes, 0..) |size, index| {
+        cursor = alignForward(cursor, surface_state_size);
+        layout.surface_offsets[index] = @intCast(cursor);
+        layout.surface_address_offsets[index] = @intCast(cursor + 8 * @sizeOf(u32));
+        cursor += surface_state_size;
+        if (size == 0)
+            return Error.EmptyBuffer;
+        if (size > std.math.maxInt(u32))
+            return Error.UnsupportedBufferSize;
+    }
+
+    const size_table_surface = buffer_sizes.len;
+    cursor = alignForward(cursor, surface_state_size);
+    layout.surface_offsets[size_table_surface] = @intCast(cursor);
+    layout.surface_address_offsets[size_table_surface] = @intCast(cursor + 8 * @sizeOf(u32));
+    cursor += surface_state_size;
+
+    cursor = alignForward(cursor, 32);
+    layout.binding_table_offset = @intCast(cursor);
+    cursor += layout.surface_count * @sizeOf(u32);
+
+    cursor = alignForward(cursor, @alignOf(u32));
+    layout.size_table_offset = @intCast(cursor);
+    cursor += @max(buffer_sizes.len, 1) * @sizeOf(u32);
+
+    cursor = alignForward(cursor, 64);
+    layout.interface_descriptor_offset = @intCast(cursor);
+    cursor += interface_descriptor_size;
+    layout.size = alignForward(cursor, page_size);
+    if (layout.size > destination.len or layout.size > page_size)
+        return Error.StateTooLarge;
+
+    @memset(destination[0..layout.size], 0);
+    @memcpy(destination[layout.kernel_offset .. layout.kernel_offset + kernel.len], kernel);
+
+    for (buffer_sizes, 0..) |size, index| {
+        _ = try encodeRawBufferSurface(destination, layout.surface_offsets[index], size);
+        putU32(destination, layout.binding_table_offset + @as(u32, @intCast(index * @sizeOf(u32))), layout.surface_offsets[index]);
+        putU32(destination, layout.size_table_offset + @as(u32, @intCast(index * @sizeOf(u32))), @intCast(size));
+    }
+    _ = try encodeRawBufferSurface(destination, layout.surface_offsets[size_table_surface], @max(buffer_sizes.len, 1) * @sizeOf(u32));
+    putU32(destination, layout.binding_table_offset + @as(u32, @intCast(size_table_surface * @sizeOf(u32))), layout.surface_offsets[size_table_surface]);
+
+    const idd = layout.interface_descriptor_offset;
+    putU32(destination, idd + 0, layout.kernel_offset);
+    putU32(destination, idd + 4, 0);
+    putU32(destination, idd + 4 * @sizeOf(u32), @as(u32, layout.surface_count) | layout.binding_table_offset);
+    putU32(destination, idd + 6 * @sizeOf(u32), 1);
+
+    return layout;
+}
+
+fn encodeRawBufferSurface(destination: []u8, offset: u32, byte_size: u64) Error!void {
+    if (byte_size == 0)
+        return Error.EmptyBuffer;
+
+    const aligned_size = std.mem.alignForward(u64, byte_size, 4);
+    const padded_size = aligned_size + (aligned_size - byte_size);
+    if (padded_size == 0 or padded_size > (@as(u64, 1) << 32))
+        return Error.UnsupportedBufferSize;
+    const length_minus_one: u32 = @intCast(padded_size - 1);
+
+    putU32(destination, offset + 0, (4 << 29) |
+        (raw_surface_format << 18) |
+        (1 << 16) |
+        (1 << 14));
+    putU32(destination, offset + 1 * @sizeOf(u32), mocs << 24);
+    putU32(destination, offset + 2 * @sizeOf(u32), (length_minus_one & 0x7f) |
+        (((length_minus_one >> 7) & 0x3fff) << 16));
+    putU32(destination, offset + 3 * @sizeOf(u32), ((length_minus_one >> 21) & 0x7ff) << 21);
+}
+
+pub const ccStatePointers = [_]u32{
+    0x780e0000,
+    0,
+};
+
+pub const pipelineSelectGpgpu = [_]u32{0x69040302};
+
+pub fn pipeControl(bits: u32) [6]u32 {
+    return .{ 0x7a000004, bits, 0, 0, 0, 0 };
+}
+
+pub const pipe_control = struct {
+    pub const state_invalidate: u32 = 1 << 2;
+    pub const constant_invalidate: u32 = 1 << 3;
+    pub const dc_flush: u32 = 1 << 5;
+    pub const texture_invalidate: u32 = 1 << 10;
+    pub const instruction_invalidate: u32 = 1 << 11;
+    pub const render_target_flush: u32 = 1 << 12;
+    pub const depth_flush: u32 = 1 << 0;
+    pub const cs_stall: u32 = 1 << 20;
+};
+
+pub fn stateBaseAddress() [19]u32 {
+    var words: [19]u32 = @splat(0);
+    words[0] = 0x61010011;
+    words[3] = mocs << 16;
+    words[4] = base_address_delta;
+    words[6] = base_address_delta;
+    words[10] = base_address_delta;
+    words[13] = (1 << 12) | 1;
+    words[15] = (1 << 12) | 1;
+    return words;
+}
+
+pub fn mediaVfeState() [9]u32 {
+    var words: [9]u32 = @splat(0);
+    words[0] = 0x70000007;
+    words[3] = (1 << 16) | (2 << 8);
+    words[5] = 2 << 16;
+    return words;
+}
+
+pub fn interfaceDescriptorLoad(offset: u32) [4]u32 {
+    return .{ 0x70020002, 0, interface_descriptor_size, offset };
+}
+
+pub fn gpgpuWalker(group_count: [3]u32, right_mask: u32) [15]u32 {
+    var words: [15]u32 = @splat(0);
+    words[0] = 0x7105000d;
+    words[7] = group_count[0];
+    words[10] = group_count[1];
+    words[12] = group_count[2];
+    words[13] = right_mask;
+    words[14] = 0xffffffff;
+    return words;
+}
+
+pub const mediaStateFlush = [_]u32{ 0x70040000, 0 };
+
+fn alignForward(value: usize, alignment: usize) usize {
+    return std.mem.alignForward(usize, value, alignment);
+}
+
+fn putU32(destination: []u8, offset: u32, value: u32) void {
+    std.mem.writeInt(u32, destination[offset..][0..@sizeOf(u32)], value, .little);
+}
+
+test "[gen9] dispatch: interface descriptor exposes internal size-table surface" {
+    var state: [page_size]u8 = undefined;
+    const layout = try writeState(&state, &.{ 0xaa, 0xbb }, &.{ 4096, 8192 });
+
+    try std.testing.expectEqual(@as(u8, 3), layout.surface_count);
+
+    const descriptor_binding_table = std.mem.readInt(
+        u32,
+        state[layout.interface_descriptor_offset + 4 * @sizeOf(u32) ..][0..@sizeOf(u32)],
+        .little,
+    );
+    try std.testing.expectEqual(layout.binding_table_offset | @as(u32, layout.surface_count), descriptor_binding_table);
+
+    for (0..layout.surface_count) |index| {
+        const entry = std.mem.readInt(
+            u32,
+            state[layout.binding_table_offset + index * @sizeOf(u32) ..][0..@sizeOf(u32)],
+            .little,
+        );
+        try std.testing.expectEqual(layout.surface_offsets[index], entry);
+    }
+}

src/intel/compiler/targets/gen9/compute/eu.zig

@@ -0,0 +1,96 @@
+pub const Opcode = enum(u7) {
+    add = 0x40,
+    addc = 0x4E,
+    @"and" = 0x05,
+    asr = 0x0C,
+    avg = 0x42,
+    bfe = 0x18,
+    bfi1 = 0x19,
+    bfi2 = 0x1A,
+    bfrev = 0x17,
+    brc = 0x23,
+    brd = 0x21,
+    @"break" = 0x28,
+    call = 0x2C,
+    calla = 0x2B,
+    cbit = 0x4D,
+    cmp = 0x10,
+    cmpn = 0x11,
+    cont = 0x29,
+    csel = 0x12,
+    dp2 = 0x57,
+    dp3 = 0x56,
+    dp4 = 0x54,
+    dph = 0x55,
+    @"else" = 0x24,
+    endif = 0x25,
+    fbh = 0x4B,
+    fbl = 0x4C,
+    frc = 0x43,
+    goto = 0x2E,
+    halt = 0x2A,
+    @"if" = 0x22,
+    illegal = 0x00,
+    jmpi = 0x20,
+    join = 0x2F,
+    line = 0x59,
+    lrp = 0x5C,
+    lzd = 0x4A,
+    mac = 0x48,
+    mach = 0x49,
+    mad = 0x5B,
+    madm = 0x5D,
+    math = 0x38,
+    mov = 0x01,
+    movi = 0x03,
+    mul = 0x41,
+    nop = 0x7E,
+    not = 0x04,
+    @"or" = 0x06,
+    pln = 0x5A,
+    ret = 0x2D,
+    rndd = 0x45,
+    rnde = 0x46,
+    rndu = 0x44,
+    rndz = 0x47,
+    sad2 = 0x50,
+    sada2 = 0x51,
+    sel = 0x02,
+    send = 0x31,
+    sendc = 0x32,
+    sends = 0x33,
+    sendsc = 0x34,
+    shl = 0x09,
+    shr = 0x08,
+    smov = 0x0A,
+    subb = 0x4F,
+    wait = 0x30,
+    @"while" = 0x27,
+    xor = 0x07,
+};
+
+pub const CompareCondition = enum(u3) {
+    none = 0,
+    zero = 1,
+    not_zero = 2,
+    greater = 3,
+    greater_or_equal = 4,
+    less = 5,
+    less_or_equal = 6,
+};
+
+pub const MathFunction = enum(u4) {
+    inv = 1,
+    log = 2,
+    exp = 3,
+    sqrt = 4,
+    rsq = 5,
+    sin = 6,
+    cos = 7,
+    sincos = 8,
+    fdiv = 9,
+    pow = 10,
+    idiv_and_rem = 11,
+    idiv = 12,
+    irem = 13,
+};

src/intel/compiler/targets/gen9/compute/eu_encoder.zig

@@ -0,0 +1,491 @@
+const std = @import("std");
+const device = @import("../../../device.zig");
+const ir_instruction = @import("../../../ir/instruction.zig");
+const operand = @import("../../../ir/operand.zig");
+const message_descriptor = @import("message_descriptor.zig");
+const eu = @import("eu.zig");
+
+pub const Error = error{
+    UnsupportedExecutionSize,
+    UnsupportedDataType,
+    UnsupportedOperand,
+    InvalidRegister,
+    InvalidRegion,
+};
+
+pub const eot_payload_grf: u8 = 112;
+
+pub const EncodedInstruction = struct {
+    words: [2]u64 = .{ 0, 0 },
+
+    pub fn setBits(self: *EncodedInstruction, high: u7, low: u7, value: u64) void {
+        const width = @as(u8, high) - @as(u8, low) + 1;
+        const word = @as(usize, high) / 64;
+        const word_low: u6 = @intCast(@as(u8, low) % 64);
+        const mask = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - width)) << word_low;
+        self.words[word] = (self.words[word] & ~mask) | ((value << word_low) & mask);
+    }
+
+    pub fn bits(self: EncodedInstruction, high: u7, low: u7) u64 {
+        const width = @as(u8, high) - @as(u8, low) + 1;
+        const word = @as(usize, high) / 64;
+        const word_low: u6 = @intCast(@as(u8, low) % 64);
+        return (self.words[word] >> word_low) & (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - width));
+    }
+};
+
+const RegisterFile = enum(u2) {
+    architecture = 0,
+    grf = 1,
+    immediate = 3,
+};
+
+const HardwareType = enum(u4) {
+    unsigned_dword = 0,
+    signed_dword = 1,
+    unsigned_word = 2,
+    float = 7,
+};
+
+const Grf = struct {
+    number: u8,
+    byte_offset: u5,
+};
+
+pub fn encodeMove(execution_size: device.ExecutionSize, move: ir_instruction.Move) Error!EncodedInstruction {
+    var encoded = try instructionHeader(.mov, execution_size);
+    const destination = try resolveGrf(move.destination.register, move.destination.region.byte_offset);
+    setDestination(&encoded, .grf, try hardwareType(move.destination.type), destination, try horizontalStride(move.destination.region.horizontal_stride));
+    try setSource0(&encoded, move.source);
+    return encoded;
+}
+
+pub fn encodeEndThread(header: operand.PhysicalGrf) Error![2]EncodedInstruction {
+    if (header.number != 0 or header.byte_offset != 0)
+        return Error.InvalidRegister;
+
+    var copy = try instructionHeader(.mov, .simd8);
+    copy.setBits(34, 34, 1); // NoMask
+    setDestination(&copy, .grf, .unsigned_dword, .{ .number = eot_payload_grf, .byte_offset = 0 }, 1);
+    copy.setBits(42, 41, @intFromEnum(RegisterFile.grf));
+    copy.setBits(46, 43, @intFromEnum(HardwareType.unsigned_dword));
+    copy.setBits(76, 69, header.number);
+    copy.setBits(81, 80, 1);
+    copy.setBits(84, 82, 3);
+    copy.setBits(88, 85, 4);
+
+    var send = try instructionHeader(.send, .simd8);
+    send.setBits(34, 34, 1); // NoMask
+    setDestination(&send, .architecture, .unsigned_word, .{ .number = 0, .byte_offset = 0 }, 1);
+    send.setBits(42, 41, @intFromEnum(RegisterFile.grf));
+    send.setBits(46, 43, @intFromEnum(HardwareType.unsigned_word));
+    send.setBits(76, 69, eot_payload_grf);
+    send.setBits(81, 80, 1);
+    send.setBits(84, 82, 3);
+    send.setBits(88, 85, 4);
+    send.setBits(90, 89, @intFromEnum(RegisterFile.immediate));
+    send.setBits(94, 91, @intFromEnum(HardwareType.unsigned_dword));
+    send.setBits(124, 96, 0x02000010); // mlen=1, no response, do not dereference URB
+    send.setBits(27, 24, 7); // Thread Spawner
+    send.setBits(127, 127, 1);
+
+    return .{ copy, send };
+}
+
+pub fn encodeSurfaceMessage(execution_size: device.ExecutionSize, message: ir_instruction.SurfaceMessage) Error!EncodedInstruction {
+    var encoded = try instructionHeader(.send, execution_size);
+    const descriptor = message_descriptor.encode(message);
+    const payload = try resolveGrf(message.payload.base, 0);
+    if (payload.byte_offset != 0)
+        return Error.InvalidRegister;
+
+    if (message.response) |response| {
+        const destination = try resolveGrf(response.base, 0);
+        if (destination.byte_offset != 0)
+            return Error.InvalidRegister;
+        setDestination(&encoded, .grf, .unsigned_word, destination, 1);
+    } else {
+        setDestination(&encoded, .architecture, .unsigned_word, .{ .number = 0, .byte_offset = 0 }, 1);
+    }
+
+    encoded.setBits(42, 41, @intFromEnum(RegisterFile.grf));
+    encoded.setBits(46, 43, @intFromEnum(HardwareType.unsigned_dword));
+    encoded.setBits(76, 69, payload.number);
+    encoded.setBits(68, 64, payload.byte_offset);
+    encoded.setBits(81, 80, 1); // horizontal stride 1
+    encoded.setBits(84, 82, 3); // width 8
+    encoded.setBits(88, 85, 4); // vertical stride 8
+    encoded.setBits(90, 89, @intFromEnum(RegisterFile.immediate));
+    encoded.setBits(94, 91, @intFromEnum(HardwareType.unsigned_dword));
+    encoded.setBits(124, 96, descriptor.value);
+    encoded.setBits(27, 24, descriptor.sfid);
+    return encoded;
+}
+
+pub fn encodeJump(displacement_bytes: i32) Error!EncodedInstruction {
+    return encodeJumpWithPredicate(displacement_bytes, null);
+}
+
+pub fn encodePredicatedJump(displacement_bytes: i32, predicate: operand.Predicate) Error!EncodedInstruction {
+    return encodeJumpWithPredicate(displacement_bytes, predicate);
+}
+
+fn encodeJumpWithPredicate(displacement_bytes: i32, predicate: ?operand.Predicate) Error!EncodedInstruction {
+    var encoded = try instructionHeader(.jmpi, .simd1);
+    encoded.setBits(34, 34, 1); // NoMask
+
+    // JMPI updates the instruction pointer: IP = IP + displacement.
+    setDestination(&encoded, .architecture, .signed_dword, .{ .number = 0xa0, .byte_offset = 0 }, 1);
+    encoded.setBits(42, 41, @intFromEnum(RegisterFile.architecture));
+    encoded.setBits(46, 43, @intFromEnum(HardwareType.signed_dword));
+    encoded.setBits(76, 69, 0xa0);
+    encoded.setBits(81, 80, 0);
+    encoded.setBits(84, 82, 0);
+    encoded.setBits(88, 85, 0);
+    setSource1Immediate(&encoded, .signed_dword, .{ .i32 = displacement_bytes });
+
+    if (predicate) |value| {
+        const flag = switch (value.flag) {
+            .physical => |physical| physical,
+            .virtual => return Error.UnsupportedOperand,
+        };
+        if (flag.register != 0 or flag.subregister > 1)
+            return Error.InvalidRegister;
+
+        encoded.setBits(19, 16, 1); // Normal predicate control.
+        encoded.setBits(20, 20, @intFromBool(value.inverse));
+        encoded.setBits(33, 33, flag.register);
+        encoded.setBits(32, 32, flag.subregister);
+    }
+
+    return encoded;
+}
+
+pub fn patchJump(encoded_bytes: []u8, displacement_bytes: i32) Error!void {
+    if (encoded_bytes.len < 16)
+        return Error.InvalidRegister;
+
+    var encoded: EncodedInstruction = .{ .words = .{
+        std.mem.readInt(u64, encoded_bytes[0..8], .little),
+        std.mem.readInt(u64, encoded_bytes[8..16], .little),
+    } };
+    if (encoded.bits(6, 0) != @intFromEnum(eu.Opcode.jmpi))
+        return Error.UnsupportedOperand;
+    encoded.setBits(127, 96, @as(u32, @bitCast(displacement_bytes)));
+    std.mem.writeInt(u64, encoded_bytes[0..8], encoded.words[0], .little);
+    std.mem.writeInt(u64, encoded_bytes[8..16], encoded.words[1], .little);
+}
+
+pub fn encodeBinary(execution_size: device.ExecutionSize, binary: ir_instruction.Binary) Error!EncodedInstruction {
+    const opcode: eu.Opcode = switch (binary.opcode) {
+        .bitwise_xor => .xor,
+        .add => .add,
+        .multiply => .mul,
+        else => return Error.UnsupportedOperand,
+    };
+
+    var encoded = try instructionHeader(opcode, execution_size);
+    const destination = try resolveGrf(binary.destination.register, binary.destination.region.byte_offset);
+    setDestination(&encoded, .grf, try hardwareType(binary.destination.type), destination, try horizontalStride(binary.destination.region.horizontal_stride));
+    try setSource0(&encoded, binary.lhs);
+    try setSource1(&encoded, binary.rhs);
+    return encoded;
+}
+
+pub fn encodeCompare(execution_size: device.ExecutionSize, compare: ir_instruction.Compare) Error!EncodedInstruction {
+    const flag = switch (compare.destination) {
+        .physical => |value| value,
+        .virtual => return Error.UnsupportedOperand,
+    };
+    if (flag.register != 0 or flag.subregister > 1)
+        return Error.InvalidRegister;
+
+    var encoded = try instructionHeader(.cmp, execution_size);
+    setDestination(&encoded, .architecture, try hardwareType(compare.lhs.type), .{ .number = 0, .byte_offset = 0 }, 1);
+    try setSource0(&encoded, compare.lhs);
+    try setSource1(&encoded, compare.rhs);
+
+    const condition: eu.CompareCondition = switch (compare.opcode) {
+        .equal => .zero,
+        .not_equal => .not_zero,
+        .greater_than => .greater,
+        .greater_or_equal => .greater_or_equal,
+        .less_than => .less,
+        .less_or_equal => .less_or_equal,
+    };
+
+    encoded.setBits(27, 24, @intFromEnum(condition));
+    encoded.setBits(33, 33, flag.register);
+    encoded.setBits(32, 32, flag.subregister);
+    return encoded;
+}
+
+pub fn encodeMath(execution_size: device.ExecutionSize, math: ir_instruction.Math) Error!EncodedInstruction {
+    if (execution_size != .simd8)
+        return Error.UnsupportedExecutionSize;
+
+    var encoded = try instructionHeader(.math, execution_size);
+    const destination = try resolveGrf(math.destination.register, math.destination.region.byte_offset);
+    setDestination(&encoded, .grf, try hardwareType(math.destination.type), destination, try horizontalStride(math.destination.region.horizontal_stride));
+
+    try setSource0(&encoded, math.lhs);
+    try setSource1(&encoded, math.rhs);
+
+    const function: eu.MathFunction = switch (math.opcode) {
+        .integer_quotient => .idiv,
+    };
+
+    encoded.setBits(27, 24, @intFromEnum(function));
+
+    return encoded;
+}
+
+fn instructionHeader(opcode: eu.Opcode, execution_size: device.ExecutionSize) Error!EncodedInstruction {
+    var encoded: EncodedInstruction = .{};
+    encoded.setBits(6, 0, @intFromEnum(opcode));
+    encoded.setBits(23, 21, try executionSize(execution_size));
+    return encoded;
+}
+
+fn setDestination(encoded: *EncodedInstruction, file: RegisterFile, data_type: HardwareType, register: Grf, horizontal_stride: u2) void {
+    encoded.setBits(36, 35, @intFromEnum(file));
+    encoded.setBits(40, 37, @intFromEnum(data_type));
+    encoded.setBits(52, 48, register.byte_offset);
+    encoded.setBits(60, 53, register.number);
+    encoded.setBits(62, 61, horizontal_stride);
+}
+
+fn setSource0Register(encoded: *EncodedInstruction, source: operand.Source, register: Grf) Error!void {
+    encoded.setBits(42, 41, @intFromEnum(RegisterFile.grf));
+    encoded.setBits(46, 43, @intFromEnum(try hardwareType(source.type)));
+    encoded.setBits(68, 64, register.byte_offset);
+    encoded.setBits(76, 69, register.number);
+    encoded.setBits(77, 77, @intFromBool(source.absolute));
+    encoded.setBits(78, 78, @intFromBool(source.negate));
+    encoded.setBits(81, 80, try horizontalStride(source.region.horizontal_stride));
+    encoded.setBits(84, 82, try regionWidth(source.region.width));
+    encoded.setBits(88, 85, try verticalStride(source.region.vertical_stride));
+}
+
+fn setSource0Immediate(encoded: *EncodedInstruction, data_type: HardwareType, immediate: operand.Immediate) void {
+    encoded.setBits(42, 41, @intFromEnum(RegisterFile.immediate));
+    encoded.setBits(46, 43, @intFromEnum(data_type));
+    encoded.setBits(90, 89, @intFromEnum(RegisterFile.architecture));
+    encoded.setBits(94, 91, @intFromEnum(data_type));
+    encoded.setBits(127, 96, switch (immediate) {
+        .u32 => |value| value,
+        .i32 => |value| @as(u32, @bitCast(value)),
+        .f32 => |value| @as(u32, @bitCast(value)),
+    });
+}
+
+fn setSource0(encoded: *EncodedInstruction, source: operand.Source) Error!void {
+    switch (source.register) {
+        .physical_grf => {
+            const register = try resolveGrf(source.register, source.region.byte_offset);
+            try setSource0Register(encoded, source, register);
+        },
+        .immediate => |immediate| {
+            setSource0Immediate(encoded, try hardwareType(source.type), try applyImmediateModifiers(immediate, source.negate, source.absolute));
+        },
+
+        else => return Error.UnsupportedOperand,
+    }
+}
+
+fn setSource1Register(encoded: *EncodedInstruction, source: operand.Source, register: Grf) Error!void {
+    encoded.setBits(90, 89, @intFromEnum(RegisterFile.grf));
+    encoded.setBits(94, 91, @intFromEnum(try hardwareType(source.type)));
+
+    // Direct addressing.
+    encoded.setBits(100, 96, register.byte_offset);
+    encoded.setBits(108, 101, register.number);
+
+    // Source modifiers.
+    encoded.setBits(109, 109, @intFromBool(source.absolute));
+    encoded.setBits(110, 110, @intFromBool(source.negate));
+
+    // AddressMode = direct.
+    encoded.setBits(111, 111, 0);
+
+    // Align1 region.
+    encoded.setBits(113, 112, try horizontalStride(source.region.horizontal_stride));
+    encoded.setBits(116, 114, try regionWidth(source.region.width));
+    encoded.setBits(120, 117, try verticalStride(source.region.vertical_stride));
+}
+
+fn setSource1Immediate(encoded: *EncodedInstruction, data_type: HardwareType, immediate: operand.Immediate) void {
+    encoded.setBits(90, 89, @intFromEnum(RegisterFile.immediate));
+
+    encoded.setBits(94, 91, @intFromEnum(data_type));
+
+    encoded.setBits(127, 96, switch (immediate) {
+        .u32 => |value| value,
+        .i32 => |value| @as(u32, @bitCast(value)),
+        .f32 => |value| @as(u32, @bitCast(value)),
+    });
+}
+
+fn setSource1(encoded: *EncodedInstruction, source: operand.Source) Error!void {
+    switch (source.register) {
+        .physical_grf => {
+            const register = try resolveGrf(source.register, source.region.byte_offset);
+            try setSource1Register(encoded, source, register);
+        },
+
+        .immediate => |immediate| {
+            setSource1Immediate(encoded, try hardwareType(source.type), try applyImmediateModifiers(immediate, source.negate, source.absolute));
+        },
+
+        else => return Error.UnsupportedOperand,
+    }
+}
+
+fn applyImmediateModifiers(immediate: operand.Immediate, negate: bool, absolute: bool) Error!operand.Immediate {
+    if (absolute)
+        return Error.UnsupportedOperand;
+    if (!negate)
+        return immediate;
+
+    return switch (immediate) {
+        .u32 => |value| .{ .u32 = 0 -% value },
+        .i32 => |value| .{ .i32 = 0 -% value },
+        .f32 => |value| .{ .f32 = -value },
+    };
+}
+
+fn resolveGrf(register: operand.RegisterRef, region_byte_offset: u16) Error!Grf {
+    const physical = switch (register) {
+        .physical_grf => |value| value,
+        else => return Error.UnsupportedOperand,
+    };
+
+    const byte_address = @as(u32, physical.number) * 32 + physical.byte_offset + region_byte_offset;
+    const number = byte_address / 32;
+
+    if (number >= 128)
+        return Error.InvalidRegister;
+
+    return .{
+        .number = @intCast(number),
+        .byte_offset = @intCast(byte_address % 32),
+    };
+}
+
+fn hardwareType(data_type: operand.DataType) Error!HardwareType {
+    return switch (data_type) {
+        .u32 => .unsigned_dword,
+        .i32 => .signed_dword,
+        .f32 => .float,
+        else => Error.UnsupportedDataType,
+    };
+}
+
+fn executionSize(size: device.ExecutionSize) Error!u3 {
+    return switch (size) {
+        .simd1 => 0,
+        .simd8 => 3,
+        else => Error.UnsupportedExecutionSize,
+    };
+}
+
+fn horizontalStride(stride: u8) Error!u2 {
+    return switch (stride) {
+        0 => 0,
+        1 => 1,
+        2 => 2,
+        4 => 3,
+        else => Error.InvalidRegion,
+    };
+}
+
+fn regionWidth(width: u8) Error!u3 {
+    return switch (width) {
+        1 => 0,
+        2 => 1,
+        4 => 2,
+        8 => 3,
+        16 => 4,
+        else => Error.InvalidRegion,
+    };
+}
+
+fn verticalStride(stride: u8) Error!u4 {
+    return switch (stride) {
+        0 => 0,
+        1 => 1,
+        2 => 2,
+        4 => 3,
+        8 => 4,
+        16 => 5,
+        32 => 6,
+        else => Error.InvalidRegion,
+    };
+}
+
+fn testBinary(opcode: ir_instruction.BinaryOpcode) ir_instruction.Binary {
+    return .{
+        .opcode = opcode,
+        .destination = .{
+            .register = .{ .physical_grf = .{ .number = 3 } },
+            .type = .u32,
+        },
+        .lhs = .{
+            .register = .{ .physical_grf = .{ .number = 1 } },
+            .type = .u32,
+            .region = operand.Region.contiguous(.simd8),
+        },
+        .rhs = .{
+            .register = .{ .immediate = .{ .u32 = 16 } },
+            .type = .u32,
+            .region = operand.Region.broadcast(),
+        },
+    };
+}
+
+test "[gen9] EU encoder: encode integer multiply" {
+    const encoded = try encodeBinary(.simd8, testBinary(.multiply));
+    try std.testing.expectEqual(@as(u64, 65), encoded.bits(6, 0));
+    try std.testing.expectEqual(@as(u64, 3), encoded.bits(23, 21));
+}
+
+test "[gen9] EU encoder: encode bitwise XOR" {
+    const encoded = try encodeBinary(.simd8, testBinary(.bitwise_xor));
+    try std.testing.expectEqual(@as(u64, 7), encoded.bits(6, 0));
+    try std.testing.expectEqual(@as(u64, 16), encoded.bits(127, 96));
+}
+
+test "[gen9] EU encoder: encode unsigned less-than comparison" {
+    const encoded = try encodeCompare(.simd8, .{
+        .opcode = .less_than,
+        .destination = .{ .physical = .{ .register = 0, .subregister = 1 } },
+        .lhs = .{
+            .register = .{ .physical_grf = .{ .number = 1 } },
+            .type = .u32,
+            .region = operand.Region.contiguous(.simd8),
+        },
+        .rhs = .{
+            .register = .{ .physical_grf = .{ .number = 2 } },
+            .type = .u32,
+            .region = operand.Region.contiguous(.simd8),
+        },
+    });
+
+    try std.testing.expectEqual(@as(u64, 16), encoded.bits(6, 0));
+    try std.testing.expectEqual(@as(u64, 5), encoded.bits(27, 24));
+    try std.testing.expectEqual(@as(u64, 0), encoded.bits(33, 33));
+    try std.testing.expectEqual(@as(u64, 1), encoded.bits(32, 32));
+}
+
+test "[gen9] EU encoder: encode predicated jump" {
+    const encoded = try encodePredicatedJump(-32, .{
+        .flag = .{ .physical = .{ .register = 0, .subregister = 1 } },
+        .inverse = true,
+    });
+
+    try std.testing.expectEqual(@as(u64, @intFromEnum(eu.Opcode.jmpi)), encoded.bits(6, 0));
+    try std.testing.expectEqual(@as(u64, 1), encoded.bits(19, 16));
+    try std.testing.expectEqual(@as(u64, 1), encoded.bits(20, 20));
+    try std.testing.expectEqual(@as(u64, 1), encoded.bits(32, 32));
+    try std.testing.expectEqual(@as(i32, -32), @as(i32, @bitCast(@as(u32, @truncate(encoded.bits(127, 96))))));
+}

src/intel/compiler/targets/gen9/compute/kernel_encoder.zig

@@ -0,0 +1,211 @@
+const std = @import("std");
+
+const eu = @import("eu_encoder.zig");
+const ids = @import("../../../ir/id.zig");
+const program_ir = @import("../../../ir/program.zig");
+
+pub const Error = std.mem.Allocator.Error || eu.Error || error{
+    InvalidProgram,
+    UnsupportedControlFlow,
+    UnsupportedOperation,
+    UnsupportedPredication,
+    EotRegisterUnavailable,
+};
+
+const JumpFixup = struct {
+    instruction_offset: usize,
+    target: ids.BlockId,
+};
+
+pub fn encode(allocator: std.mem.Allocator, program: *program_ir.Program) Error![]u8 {
+    if (!program.properties.registers_allocated)
+        return Error.InvalidProgram;
+    if (program.program_data.total_grf_count > eu.eot_payload_grf)
+        return Error.EotRegisterUnavailable;
+
+    const entry_id = program.entry_block orelse return Error.InvalidProgram;
+    if (!program.blocks.isLive(entry_id))
+        return Error.InvalidProgram;
+
+    const block_offsets = try allocator.alloc(?usize, program.blocks.entries.items.len);
+    defer allocator.free(block_offsets);
+    @memset(block_offsets, null);
+
+    var block_order: std.ArrayList(ids.BlockId) = .empty;
+    defer block_order.deinit(allocator);
+    try block_order.append(allocator, entry_id);
+    for (program.blocks.entries.items, 0..) |block, block_index| {
+        if (block != null and block_index != entry_id.index())
+            try block_order.append(allocator, ids.BlockId.fromIndex(block_index));
+    }
+
+    var fixups: std.ArrayList(JumpFixup) = .empty;
+    defer fixups.deinit(allocator);
+    var kernel: std.ArrayList(u8) = .empty;
+    errdefer kernel.deinit(allocator);
+
+    for (block_order.items) |block_id| {
+        const block = program.blocks.get(block_id) orelse return Error.InvalidProgram;
+        block_offsets[block_id.index()] = kernel.items.len;
+
+        for (block.instructions.items) |instruction_id|
+            try encodeInstruction(allocator, &kernel, program, instruction_id);
+
+        const terminator = block.terminator orelse return Error.InvalidProgram;
+        switch (terminator) {
+            .jump => |edge| {
+                const instruction_offset = kernel.items.len;
+                try appendInstruction(allocator, &kernel, try eu.encodeJump(0));
+                try fixups.append(allocator, .{
+                    .instruction_offset = instruction_offset,
+                    .target = edge.target,
+                });
+            },
+            .conditional_branch => |branch| {
+                const true_instruction_offset = kernel.items.len;
+                try appendInstruction(allocator, &kernel, try eu.encodePredicatedJump(0, branch.predicate));
+                try fixups.append(allocator, .{
+                    .instruction_offset = true_instruction_offset,
+                    .target = branch.true_edge.target,
+                });
+
+                const false_instruction_offset = kernel.items.len;
+                try appendInstruction(allocator, &kernel, try eu.encodeJump(0));
+                try fixups.append(allocator, .{
+                    .instruction_offset = false_instruction_offset,
+                    .target = branch.false_edge.target,
+                });
+            },
+            .end_thread => {
+                const header = program.payload.header_grf orelse return Error.InvalidProgram;
+                const instructions = try eu.encodeEndThread(header);
+                for (instructions) |encoded|
+                    try appendInstruction(allocator, &kernel, encoded);
+                program.program_data.total_grf_count = eu.eot_payload_grf + 1;
+            },
+            .@"unreachable" => return Error.UnsupportedControlFlow,
+        }
+    }
+
+    for (fixups.items) |fixup| {
+        if (fixup.target.index() >= block_offsets.len)
+            return Error.InvalidProgram;
+        const target_offset = block_offsets[fixup.target.index()] orelse return Error.InvalidProgram;
+        const next_instruction_offset = fixup.instruction_offset + 16;
+        const displacement = std.math.cast(i32, @as(i64, @intCast(target_offset)) - @as(i64, @intCast(next_instruction_offset))) orelse
+            return Error.UnsupportedControlFlow;
+        try eu.patchJump(kernel.items[fixup.instruction_offset..][0..16], displacement);
+    }
+
+    return kernel.toOwnedSlice(allocator);
+}
+
+fn encodeInstruction(allocator: std.mem.Allocator, kernel: *std.ArrayList(u8), program: *const program_ir.Program, instruction_id: ids.InstructionId) Error!void {
+    const inst = program.instructions.get(instruction_id) orelse return Error.InvalidProgram;
+    if (inst.predicate != null) {
+        std.log.scoped(.FlintEuEncoder).err("cannot encode instruction {d} ({t}): predication is not supported", .{ instruction_id.index(), std.meta.activeTag(inst.operation) });
+        return Error.UnsupportedPredication;
+    }
+
+    const encoded = switch (inst.operation) {
+        .move => |move| eu.encodeMove(inst.execution_size, move),
+        .surface_message => |message| eu.encodeSurfaceMessage(inst.execution_size, message),
+        .binary => |binary| eu.encodeBinary(inst.execution_size, binary),
+        .compare => |compare| eu.encodeCompare(inst.execution_size, compare),
+        .math => |math| eu.encodeMath(inst.execution_size, math),
+        else => {
+            std.log.scoped(.FlintEuEncoder).err("cannot encode instruction {d}: unsupported operation {t}", .{ instruction_id.index(), std.meta.activeTag(inst.operation) });
+            return Error.UnsupportedOperation;
+        },
+    } catch |err| {
+        std.log.scoped(.FlintEuEncoder).err("failed to encode instruction {d} ({t}): {s}", .{ instruction_id.index(), std.meta.activeTag(inst.operation), @errorName(err) });
+        if (err == error.InvalidRegion) switch (inst.operation) {
+            .move => |move| std.log.scoped(.FlintEuEncoder).err("move in block {d}: destination {t} byte={d} hstride={d}; source {t} byte={d} vstride={d} width={d} hstride={d}", .{
+                inst.parent_block.index(),
+                move.destination.register,
+                move.destination.region.byte_offset,
+                move.destination.region.horizontal_stride,
+                move.source.register,
+                move.source.region.byte_offset,
+                move.source.region.vertical_stride,
+                move.source.region.width,
+                move.source.region.horizontal_stride,
+            }),
+            else => {},
+        };
+        return err;
+    };
+    try appendInstruction(allocator, kernel, encoded);
+}
+
+fn appendInstruction(allocator: std.mem.Allocator, kernel: *std.ArrayList(u8), instruction: eu.EncodedInstruction) std.mem.Allocator.Error!void {
+    var bytes: [16]u8 = undefined;
+    std.mem.writeInt(u64, bytes[0..8], instruction.words[0], .little);
+    std.mem.writeInt(u64, bytes[8..16], instruction.words[1], .little);
+    try kernel.appendSlice(allocator, &bytes);
+}
+
+test "[gen9] kernel encoder: patch unconditional jump between blocks" {
+    const device = @import("../../../device.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+
+    const entry = try program.addBlock("entry");
+    const exit = try program.addBlock("exit");
+    try program.setEntryBlock(entry);
+    try program.setTerminator(entry, .{ .jump = .{ .target = exit, .arguments = &.{} } });
+    try program.setTerminator(exit, .end_thread);
+    program.payload.header_grf = .{ .number = 0 };
+    program.properties.registers_allocated = true;
+
+    const kernel = try encode(std.testing.allocator, &program);
+    defer std.testing.allocator.free(kernel);
+
+    try std.testing.expectEqual(@as(usize, 48), kernel.len);
+    try std.testing.expectEqual(@as(u7, 32), @as(u7, @truncate(std.mem.readInt(u64, kernel[0..8], .little))));
+    try std.testing.expectEqual(@as(i32, 0), @as(i32, @bitCast(std.mem.readInt(u32, kernel[12..16], .little))));
+}
+
+test "[gen9] kernel encoder: patch conditional branch targets" {
+    const device = @import("../../../device.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+
+    const entry = try program.addBlock("entry");
+    const true_block = try program.addBlock("true");
+    const false_block = try program.addBlock("false");
+    try program.setEntryBlock(entry);
+    try program.setTerminator(entry, .{ .conditional_branch = .{
+        .predicate = .{ .flag = .{ .physical = .{ .register = 0, .subregister = 1 } } },
+        .true_edge = .{ .target = true_block, .arguments = &.{} },
+        .false_edge = .{ .target = false_block, .arguments = &.{} },
+    } });
+    try program.setTerminator(true_block, .end_thread);
+    try program.setTerminator(false_block, .end_thread);
+    program.payload.header_grf = .{ .number = 0 };
+    program.properties.registers_allocated = true;
+
+    const kernel = try encode(std.testing.allocator, &program);
+    defer std.testing.allocator.free(kernel);
+
+    try std.testing.expectEqual(@as(usize, 96), kernel.len);
+    try std.testing.expectEqual(@as(i32, 16), @as(i32, @bitCast(std.mem.readInt(u32, kernel[12..16], .little))));
+    try std.testing.expectEqual(@as(i32, 32), @as(i32, @bitCast(std.mem.readInt(u32, kernel[28..32], .little))));
+    const first_word = std.mem.readInt(u64, kernel[0..8], .little);
+    try std.testing.expectEqual(@as(u64, 1), (first_word >> 16) & 0xf);
+    try std.testing.expectEqual(@as(u64, 1), (first_word >> 32) & 0x1);
+}

src/intel/compiler/targets/gen9/compute/message_addresses.zig

@@ -0,0 +1,211 @@
+const std = @import("std");
+
+const Builder = @import("../../../ir/Builder.zig");
+const ids = @import("../../../ir/id.zig");
+const operand = @import("../../../ir/operand.zig");
+const program_ir = @import("../../../ir/program.zig");
+const instruction = @import("../../../ir/instruction.zig");
+
+pub const Error = std.mem.Allocator.Error || error{
+    MessagesNotLowered,
+    InvalidProgram,
+};
+
+const AddressAdjustment = struct {
+    address: operand.Source,
+    immediate_offset: u32,
+};
+
+pub fn run(program: *program_ir.Program) Error!void {
+    if (!program.properties.messages_lowered)
+        return Error.MessagesNotLowered;
+    if (program.properties.message_addresses_lowered)
+        return;
+
+    var builder = Builder.init(program);
+    for (program.blocks.entries.items, 0..) |entry, block_index| {
+        _ = entry orelse continue;
+        const block_id = ids.BlockId.fromIndex(block_index);
+        var instruction_index: usize = 0;
+
+        while (true) {
+            const block = program.blocks.get(block_id) orelse return Error.InvalidProgram;
+            if (instruction_index >= block.instructions.items.len)
+                break;
+
+            const instruction_id = block.instructions.items[instruction_index];
+            const inst = program.instructions.get(instruction_id) orelse return Error.InvalidProgram;
+            const adjustment = addressAdjustment(inst.operation) orelse {
+                instruction_index += 1;
+                continue;
+            };
+            if (adjustment.address.type != .u32)
+                return Error.InvalidProgram;
+
+            if (adjustment.immediate_offset == 0) {
+                instruction_index += 1;
+                continue;
+            }
+
+            switch (adjustment.address.register) {
+                .immediate => |immediate| {
+                    const base = switch (immediate) {
+                        .u32 => |value| value,
+                        else => return Error.InvalidProgram,
+                    };
+                    const mutable = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
+                    const address = messageAddressMut(&mutable.operation) orelse return Error.InvalidProgram;
+                    address.source.register = .{ .immediate = .{ .u32 = base +% adjustment.immediate_offset } };
+                    address.immediate_offset.* = 0;
+                    instruction_index += 1;
+                },
+                .virtual,
+                .physical_grf,
+                .architecture,
+                => {
+                    const execution_width: u32 = @intFromEnum(inst.execution_size);
+                    const size_bytes = execution_width * @sizeOf(u32);
+                    const address_register = builder.addVirtualRegister(.{
+                        .size_bytes = size_bytes,
+                        .alignment_bytes = @intCast(@min(size_bytes, program.device_info.grf_size_bytes)),
+                        .element_type = .u32,
+                        .lane_count = @intCast(execution_width),
+                        .class = .temporary,
+                    }) catch |err| return mapBuilderError(err);
+
+                    _ = builder.insertInstruction(block_id, instruction_index, inst.execution_size, inst.predicate, .{
+                        .binary = .{
+                            .opcode = .add,
+                            .destination = .{
+                                .register = .{ .virtual = address_register },
+                                .type = .u32,
+                            },
+                            .lhs = adjustment.address,
+                            .rhs = immediateSource(adjustment.immediate_offset),
+                        },
+                    }) catch |err| return mapBuilderError(err);
+
+                    const mutable = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
+                    const address = messageAddressMut(&mutable.operation) orelse return Error.InvalidProgram;
+                    address.source.* = .{
+                        .register = .{ .virtual = address_register },
+                        .type = .u32,
+                        .region = operand.Region.contiguous(inst.execution_size),
+                    };
+                    address.immediate_offset.* = 0;
+                    instruction_index += 2;
+                },
+                .null => return Error.InvalidProgram,
+            }
+        }
+    }
+
+    program.properties.message_addresses_lowered = true;
+}
+
+fn addressAdjustment(operation: instruction.Operation) ?AddressAdjustment {
+    return switch (operation) {
+        .surface_read => |op| .{ .address = op.address, .immediate_offset = op.immediate_offset },
+        .surface_write => |op| .{ .address = op.address, .immediate_offset = op.immediate_offset },
+        else => null,
+    };
+}
+
+const MutableAddress = struct {
+    source: *operand.Source,
+    immediate_offset: *u32,
+};
+
+fn messageAddressMut(operation: *instruction.Operation) ?MutableAddress {
+    return switch (operation.*) {
+        .surface_read => |*op| .{ .source = &op.address, .immediate_offset = &op.immediate_offset },
+        .surface_write => |*op| .{ .source = &op.address, .immediate_offset = &op.immediate_offset },
+        else => null,
+    };
+}
+
+fn immediateSource(value: u32) operand.Source {
+    return .{
+        .register = .{ .immediate = .{ .u32 = value } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    };
+}
+
+fn mapBuilderError(err: Builder.Error) Error {
+    return switch (err) {
+        error.OutOfMemory => Error.OutOfMemory,
+        else => Error.InvalidProgram,
+    };
+}
+
+const device = @import("../../../device.zig");
+
+const test_device: device.DeviceInfo = .{
+    .generation = .gen9,
+    .platform = .skylake,
+    .pci_device_id = 0x1912,
+    .grf_count = 128,
+};
+
+fn markPrerequisite(program: *program_ir.Program) void {
+    program.properties.messages_lowered = true;
+}
+
+test "[gen9] message addresses: fold immediate offsets" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    const entry = try program.addBlock("entry");
+    const message = try program.appendInstruction(entry, .simd8, null, .{ .surface_write = .{
+        .binding_table = 0,
+        .address = immediateSource(12),
+        .immediate_offset = 4,
+        .data = immediateSource(7),
+    } });
+    try program.setTerminator(entry, .end_thread);
+    markPrerequisite(&program);
+
+    try run(&program);
+
+    const write = program.instructions.get(message).?.operation.surface_write;
+    try std.testing.expectEqual(@as(u32, 16), write.address.register.immediate.u32);
+    try std.testing.expectEqual(@as(u32, 0), write.immediate_offset);
+    try std.testing.expect(program.properties.message_addresses_lowered);
+}
+
+test "[gen9] message addresses: materialize dynamic offsets" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    const base = try program.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const entry = try program.addBlock("entry");
+    const message = try program.appendInstruction(entry, .simd8, null, .{ .surface_read = .{
+        .destination = .{ .register = .{ .virtual = base }, .type = .u32 },
+        .binding_table = 0,
+        .address = .{
+            .register = .{ .virtual = base },
+            .type = .u32,
+            .region = operand.Region.contiguous(.simd8),
+        },
+        .immediate_offset = 8,
+    } });
+    try program.setTerminator(entry, .end_thread);
+    markPrerequisite(&program);
+
+    try run(&program);
+
+    const block = program.blocks.get(entry).?;
+    try std.testing.expectEqual(@as(usize, 2), block.instructions.items.len);
+    try std.testing.expect(program.instructions.get(block.instructions.items[0]).?.operation == .binary);
+    const read = program.instructions.get(message).?.operation.surface_read;
+    try std.testing.expect(read.address.register == .virtual);
+    try std.testing.expect(read.address.register.virtual != base);
+    try std.testing.expectEqual(@as(u32, 0), read.immediate_offset);
+}

src/intel/compiler/targets/gen9/compute/message_descriptor.zig

@@ -0,0 +1,90 @@
+const instruction = @import("../../../ir/instruction.zig");
+
+pub const Descriptor = struct {
+    sfid: u8,
+    value: u32,
+    message_length: u8,
+    response_length: u8,
+};
+
+const dc1_sfid: u8 = 12;
+const simd8_one_channel_control: u8 = 0x2e;
+
+const MessageType = enum(u8) {
+    untyped_surface_read = 1,
+    untyped_surface_write = 9,
+};
+
+pub fn encode(message: instruction.SurfaceMessage) Descriptor {
+    const lengths: struct { message: u8, response: u8 } = switch (message.kind) {
+        .read => .{ .message = 1, .response = 1 },
+        .write => .{ .message = 2, .response = 0 },
+    };
+    const message_type: MessageType = switch (message.kind) {
+        .read => .untyped_surface_read,
+        .write => .untyped_surface_write,
+    };
+
+    return .{
+        .sfid = dc1_sfid,
+        .value = makeDescriptor(
+            message.binding_table,
+            simd8_one_channel_control,
+            message_type,
+            lengths.message,
+            lengths.response,
+        ),
+        .message_length = lengths.message,
+        .response_length = lengths.response,
+    };
+}
+
+fn makeDescriptor(binding_table: u8, message_control: u8, message_type: MessageType, message_length: u8, response_length: u8) u32 {
+    return @as(u32, binding_table) |
+        (@as(u32, message_control) << 8) |
+        (@as(u32, @intFromEnum(message_type)) << 14) |
+        (@as(u32, response_length) << 20) |
+        (@as(u32, message_length) << 25);
+}
+
+test "[gen9] message descriptor: encode SIMD8 one-channel surface read" {
+    const std = @import("std");
+    const descriptor = encode(.{
+        .kind = .read,
+        .binding_table = 3,
+        .payload = .{ .base = .{ .physical_grf = .{ .number = 1 } }, .register_count = 1 },
+        .response = .{ .base = .{ .physical_grf = .{ .number = 2 } }, .register_count = 1 },
+        .data_type = .u32,
+    });
+
+    try std.testing.expectEqual(@as(u8, 12), descriptor.sfid);
+    try std.testing.expectEqual(@as(u8, 1), descriptor.message_length);
+    try std.testing.expectEqual(@as(u8, 1), descriptor.response_length);
+    try std.testing.expectEqual(@as(u8, 3), @as(u8, @truncate(descriptor.value)));
+    try std.testing.expectEqual(@as(u8, 0x2e), @as(u8, @truncate(descriptor.value >> 8)) & 0x3f);
+    try std.testing.expectEqual(@as(u8, 1), @as(u8, @truncate(descriptor.value >> 14)) & 0x1f);
+    try std.testing.expectEqual(@as(u8, 1), @as(u8, @truncate(descriptor.value >> 20)) & 0x1f);
+    try std.testing.expectEqual(@as(u8, 1), @as(u8, @truncate(descriptor.value >> 25)) & 0x0f);
+    try std.testing.expectEqual(@as(u32, 0x02106e03), descriptor.value);
+}
+
+test "[gen9] message descriptor: encode SIMD8 one-channel surface write" {
+    const std = @import("std");
+    const descriptor = encode(.{
+        .kind = .write,
+        .binding_table = 7,
+        .payload = .{ .base = .{ .physical_grf = .{ .number = 1 } }, .register_count = 2 },
+        .response = null,
+        .data_type = .u32,
+    });
+
+    try std.testing.expectEqual(@as(u8, 12), descriptor.sfid);
+    try std.testing.expectEqual(@as(u8, 2), descriptor.message_length);
+    try std.testing.expectEqual(@as(u8, 0), descriptor.response_length);
+    try std.testing.expectEqual(@as(u8, 7), @as(u8, @truncate(descriptor.value)));
+    try std.testing.expectEqual(@as(u8, 0x2e), @as(u8, @truncate(descriptor.value >> 8)) & 0x3f);
+    try std.testing.expectEqual(@as(u8, 9), @as(u8, @truncate(descriptor.value >> 14)) & 0x1f);
+    try std.testing.expectEqual(@as(u8, 0), @as(u8, @truncate(descriptor.value >> 20)) & 0x1f);
+    try std.testing.expectEqual(@as(u8, 2), @as(u8, @truncate(descriptor.value >> 25)) & 0x0f);
+    try std.testing.expectEqual(@as(u32, 0x04026e07), descriptor.value);
+}

src/intel/compiler/targets/gen9/compute/message_lowering.zig

@@ -0,0 +1,110 @@
+const instruction = @import("../../../ir/instruction.zig");
+const operand = @import("../../../ir/operand.zig");
+const program_ir = @import("../../../ir/program.zig");
+
+pub const Error = error{
+    ResourcesNotLowered,
+    InvalidProgram,
+};
+
+pub fn run(program: *program_ir.Program) Error!void {
+    if (!program.properties.resources_lowered)
+        return Error.ResourcesNotLowered;
+    if (program.properties.messages_lowered)
+        return;
+
+    for (program.instructions.entries.items) |*entry| {
+        const inst = if (entry.*) |*value| value else continue;
+        inst.operation = switch (inst.operation) {
+            .load_buffer => |op| .{ .surface_read = .{
+                .destination = op.destination,
+                .binding_table = bindingTableIndex(op.buffer) orelse return Error.InvalidProgram,
+                .address = op.byte_offset,
+                .immediate_offset = op.immediate_offset,
+            } },
+            .store_buffer => |op| .{ .surface_write = .{
+                .binding_table = bindingTableIndex(op.buffer) orelse return Error.InvalidProgram,
+                .address = op.byte_offset,
+                .immediate_offset = op.immediate_offset,
+                .data = op.source,
+            } },
+            else => inst.operation,
+        };
+    }
+
+    program.properties.messages_lowered = true;
+}
+
+fn bindingTableIndex(reference: instruction.BufferReference) ?u8 {
+    return switch (reference) {
+        .binding_table => |index| index,
+        .logical => null,
+    };
+}
+
+const std = @import("std");
+const device = @import("../../../device.zig");
+
+const test_device: device.DeviceInfo = .{
+    .generation = .gen9,
+    .platform = .skylake,
+    .pci_device_id = 0x1912,
+    .grf_count = 128,
+};
+
+fn immediate(value: u32) operand.Source {
+    return .{
+        .register = .{ .immediate = .{ .u32 = value } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    };
+}
+
+test "[gen9] compute message lowering: select surface messages" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    const value = try program.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const entry = try program.addBlock("entry");
+    const load = try program.appendInstruction(entry, .simd8, null, .{ .load_buffer = .{
+        .destination = .{ .register = .{ .virtual = value }, .type = .u32 },
+        .buffer = .{ .binding_table = 2 },
+        .byte_offset = immediate(16),
+        .immediate_offset = 4,
+    } });
+    const store = try program.appendInstruction(entry, .simd8, null, .{ .store_buffer = .{
+        .buffer = .{ .binding_table = 3 },
+        .byte_offset = immediate(32),
+        .immediate_offset = 8,
+        .source = .{
+            .register = .{ .virtual = value },
+            .type = .u32,
+            .region = operand.Region.contiguous(.simd8),
+        },
+    } });
+    try program.setTerminator(entry, .end_thread);
+    program.properties.resources_lowered = true;
+
+    try run(&program);
+
+    const read = program.instructions.get(load).?.operation.surface_read;
+    try std.testing.expectEqual(@as(u8, 2), read.binding_table);
+    try std.testing.expectEqual(@as(u32, 4), read.immediate_offset);
+    const write = program.instructions.get(store).?.operation.surface_write;
+    try std.testing.expectEqual(@as(u8, 3), write.binding_table);
+    try std.testing.expectEqual(@as(u32, 8), write.immediate_offset);
+    try std.testing.expect(program.properties.messages_lowered);
+}
+
+test "[gen9] compute message lowering: reject unresolved resources" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    try std.testing.expectError(Error.ResourcesNotLowered, run(&program));
+}

src/intel/compiler/targets/gen9/compute/message_payloads.zig

@@ -0,0 +1,200 @@
+const std = @import("std");
+
+const Builder = @import("../../../ir/Builder.zig");
+const device = @import("../../../device.zig");
+const ids = @import("../../../ir/id.zig");
+const instruction = @import("../../../ir/instruction.zig");
+const operand = @import("../../../ir/operand.zig");
+const program_ir = @import("../../../ir/program.zig");
+
+pub const Error = std.mem.Allocator.Error || error{
+    MessageAddressesNotLowered,
+    InvalidProgram,
+};
+
+pub fn run(program: *program_ir.Program) Error!void {
+    if (!program.properties.message_addresses_lowered)
+        return Error.MessageAddressesNotLowered;
+    if (program.properties.message_payloads_lowered)
+        return;
+    if (program.device_info.grf_size_bytes != 32)
+        return Error.InvalidProgram;
+
+    var builder = Builder.init(program);
+    for (program.blocks.entries.items, 0..) |entry, block_index| {
+        _ = entry orelse continue;
+        const block_id = ids.BlockId.fromIndex(block_index);
+        var instruction_index: usize = 0;
+
+        while (true) {
+            const block = program.blocks.get(block_id) orelse return Error.InvalidProgram;
+            if (instruction_index >= block.instructions.items.len)
+                break;
+
+            const instruction_id = block.instructions.items[instruction_index];
+            const inst = program.instructions.get(instruction_id) orelse return Error.InvalidProgram;
+            const execution_size = inst.execution_size;
+            switch (inst.operation) {
+                .surface_read => |op| {
+                    if (op.immediate_offset != 0 or op.address.type != .u32)
+                        return Error.InvalidProgram;
+                    const response = try responseSpan(op.destination);
+                    const payload = try addPayloadRegister(&builder, execution_size, 1);
+                    _ = builder.insertInstruction(block_id, instruction_index, execution_size, null, .{ .move = .{
+                        .destination = payloadDestination(payload, 0, .u32),
+                        .source = op.address,
+                    } }) catch |err| return mapBuilderError(err);
+
+                    const mutable = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
+                    mutable.operation = .{ .surface_message = .{
+                        .kind = .read,
+                        .binding_table = op.binding_table,
+                        .payload = .{ .base = .{ .virtual = payload }, .register_count = 1 },
+                        .response = response,
+                        .data_type = op.destination.type,
+                    } };
+                    instruction_index += 2;
+                },
+                .surface_write => |op| {
+                    if (op.immediate_offset != 0 or op.address.type != .u32)
+                        return Error.InvalidProgram;
+                    const payload = try addPayloadRegister(&builder, execution_size, 2);
+                    _ = builder.insertInstruction(block_id, instruction_index, execution_size, null, .{ .move = .{
+                        .destination = payloadDestination(payload, 0, .u32),
+                        .source = op.address,
+                    } }) catch |err| return mapBuilderError(err);
+                    _ = builder.insertInstruction(block_id, instruction_index + 1, execution_size, null, .{ .move = .{
+                        .destination = payloadDestination(payload, 32, op.data.type),
+                        .source = op.data,
+                    } }) catch |err| return mapBuilderError(err);
+
+                    const mutable = program.instructions.getMut(instruction_id) orelse return Error.InvalidProgram;
+                    mutable.operation = .{ .surface_message = .{
+                        .kind = .write,
+                        .binding_table = op.binding_table,
+                        .payload = .{ .base = .{ .virtual = payload }, .register_count = 2 },
+                        .response = null,
+                        .data_type = op.data.type,
+                    } };
+                    instruction_index += 3;
+                },
+                else => instruction_index += 1,
+            }
+        }
+    }
+
+    program.properties.message_payloads_lowered = true;
+}
+
+fn addPayloadRegister(builder: *Builder, execution_size: device.ExecutionSize, register_count: u8) Error!ids.VirtualRegisterId {
+    return builder.addVirtualRegister(.{
+        .size_bytes = @as(u32, register_count) * 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = @intFromEnum(execution_size),
+        .class = .temporary,
+        .spillable = false,
+    }) catch |err| return mapBuilderError(err);
+}
+
+fn payloadDestination(register: ids.VirtualRegisterId, byte_offset: u16, data_type: operand.DataType) operand.Destination {
+    return .{
+        .register = .{ .virtual = register },
+        .type = data_type,
+        .region = .{ .byte_offset = byte_offset },
+    };
+}
+
+fn responseSpan(destination: operand.Destination) Error!operand.RegisterSpan {
+    if (destination.region.byte_offset != 0 or destination.region.horizontal_stride != 1)
+        return Error.InvalidProgram;
+    return switch (destination.register) {
+        .virtual, .physical_grf => .{
+            .base = destination.register,
+            .register_count = 1,
+        },
+        else => Error.InvalidProgram,
+    };
+}
+
+fn mapBuilderError(err: Builder.Error) Error {
+    return switch (err) {
+        error.OutOfMemory => Error.OutOfMemory,
+        else => Error.InvalidProgram,
+    };
+}
+
+const test_device: device.DeviceInfo = .{
+    .generation = .gen9,
+    .platform = .skylake,
+    .pci_device_id = 0x1912,
+    .grf_count = 128,
+};
+
+fn immediate(value: u32) operand.Source {
+    return .{
+        .register = .{ .immediate = .{ .u32 = value } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    };
+}
+
+test "[gen9] message payloads: pack surface write address and data" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    const entry = try program.addBlock("entry");
+    const message = try program.appendInstruction(entry, .simd8, null, .{ .surface_write = .{
+        .binding_table = 2,
+        .address = immediate(16),
+        .data = immediate(42),
+    } });
+    try program.setTerminator(entry, .end_thread);
+    program.properties.message_addresses_lowered = true;
+
+    try run(&program);
+
+    const block = program.blocks.get(entry).?;
+    try std.testing.expectEqual(@as(usize, 3), block.instructions.items.len);
+    const address_move = program.instructions.get(block.instructions.items[0]).?.operation.move;
+    const data_move = program.instructions.get(block.instructions.items[1]).?.operation.move;
+    try std.testing.expectEqual(@as(u16, 0), address_move.destination.region.byte_offset);
+    try std.testing.expectEqual(@as(u16, 32), data_move.destination.region.byte_offset);
+    try std.testing.expectEqual(address_move.destination.register.virtual, data_move.destination.register.virtual);
+
+    const send = program.instructions.get(message).?.operation.surface_message;
+    try std.testing.expectEqual(instruction.SurfaceMessageKind.write, send.kind);
+    try std.testing.expectEqual(@as(u8, 2), send.binding_table);
+    try std.testing.expectEqual(@as(u8, 2), send.payload.register_count);
+    try std.testing.expect(send.response == null);
+    try std.testing.expect(program.properties.message_payloads_lowered);
+}
+
+test "[gen9] message payloads: prepare surface read response" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    const result = try program.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .response,
+    });
+    const entry = try program.addBlock("entry");
+    const message = try program.appendInstruction(entry, .simd8, null, .{ .surface_read = .{
+        .destination = .{ .register = .{ .virtual = result }, .type = .u32 },
+        .binding_table = 1,
+        .address = immediate(0),
+    } });
+    try program.setTerminator(entry, .end_thread);
+    program.properties.message_addresses_lowered = true;
+
+    try run(&program);
+
+    const send = program.instructions.get(message).?.operation.surface_message;
+    try std.testing.expectEqual(instruction.SurfaceMessageKind.read, send.kind);
+    try std.testing.expectEqual(@as(u8, 1), send.payload.register_count);
+    try std.testing.expectEqual(result, send.response.?.base.virtual);
+    try std.testing.expectEqual(@as(u8, 1), send.response.?.register_count);
+}

src/intel/compiler/targets/gen9/compute/pipeline.zig

@@ -0,0 +1,120 @@
+const std = @import("std");
+const shader_ir = @import("shader_ir").ir;
+
+const device = @import("../../../device.zig");
+const program_ir = @import("../../../ir/program.zig");
+const common_ir = @import("../../../lower/common_ir.zig");
+const block_arguments = @import("../../../lower/block_arguments.zig");
+const parallel_copies = @import("../../../lower/parallel_copies.zig");
+const flag_allocation = @import("../flag_allocation.zig");
+const register_allocation = @import("../register_allocation.zig");
+
+const compute = @import("compute.zig");
+const abi = @import("abi.zig");
+const array_length_lowering = @import("array_length_lowering.zig");
+const kernel_encoder = @import("kernel_encoder.zig");
+const message_addresses = @import("message_addresses.zig");
+const message_lowering = @import("message_lowering.zig");
+const message_payloads = @import("message_payloads.zig");
+const resource_layout = @import("resource_layout.zig");
+const resource_lowering = @import("resource_lowering.zig");
+const regions = @import("regions.zig");
+const system_values = @import("system_values.zig");
+
+pub const Error = common_ir.Error ||
+    block_arguments.Error ||
+    parallel_copies.Error ||
+    abi.Error ||
+    array_length_lowering.Error ||
+    kernel_encoder.Error ||
+    message_addresses.Error ||
+    message_lowering.Error ||
+    message_payloads.Error ||
+    resource_layout.Error ||
+    resource_lowering.Error ||
+    system_values.Error ||
+    flag_allocation.Error ||
+    register_allocation.Error ||
+    compute.Error ||
+    error{
+        UnsupportedGeneration,
+        UnsupportedStage,
+        UnsupportedDispatchWidth,
+        UnsupportedGrfSize,
+    };
+
+pub const Artifact = struct {
+    program: program_ir.Program,
+    resources: resource_layout.Layout,
+    kernel: ?[]u8,
+
+    pub fn deinit(self: *Artifact, allocator: std.mem.Allocator) void {
+        if (self.kernel) |kernel|
+            allocator.free(kernel);
+        self.resources.deinit(allocator);
+        self.program.deinit();
+        self.* = undefined;
+    }
+};
+
+pub fn compile(allocator: std.mem.Allocator, module: *shader_ir.module.Module, device_info: device.DeviceInfo, options: common_ir.Options) Error!Artifact {
+    if (device_info.generation != .gen9)
+        return Error.UnsupportedGeneration;
+    if (module.stage != .compute)
+        return Error.UnsupportedStage;
+    if (options.dispatch_width != .simd8 or !device_info.supportsDispatch(.simd8))
+        return Error.UnsupportedDispatchWidth;
+    if (device_info.grf_size_bytes != 32)
+        return Error.UnsupportedGrfSize;
+    if (module.execution_modes.workgroup_size) |workgroup_size|
+        try compute.validateWorkgroupSize(workgroup_size);
+
+    var program = try common_ir.lower(
+        allocator,
+        module,
+        device_info,
+        options,
+    );
+    errdefer program.deinit();
+
+    try abi.run(&program);
+    try system_values.run(&program);
+    try block_arguments.run(allocator, &program);
+    try parallel_copies.run(allocator, &program);
+
+    var resources = try resource_layout.Layout.init(allocator, &program);
+    errdefer resources.deinit(allocator);
+
+    try resource_lowering.run(&program, &resources);
+    try array_length_lowering.run(&program, &resources);
+    try message_lowering.run(&program);
+    try message_addresses.run(&program);
+    try message_payloads.run(&program);
+    regions.run(&program);
+    try flag_allocation.run(allocator, &program);
+    try register_allocation.run(allocator, &program);
+
+    const kernel = kernel_encoder.encode(allocator, &program) catch |err| encoding_error: {
+        std.log.scoped(.FlintCompiler).err("Gen9 EU kernel encoding failed: {s}", .{@errorName(err)});
+        break :encoding_error switch (err) {
+            error.UnsupportedControlFlow,
+            error.UnsupportedOperation,
+            error.UnsupportedPredication,
+            error.UnsupportedExecutionSize,
+            error.UnsupportedDataType,
+            error.UnsupportedOperand,
+            error.InvalidRegister,
+            error.InvalidRegion,
+            error.EotRegisterUnavailable,
+            => null,
+            else => return err,
+        };
+    };
+    errdefer if (kernel) |bytes| allocator.free(bytes);
+
+    return .{
+        .program = program,
+        .resources = resources,
+        .kernel = kernel,
+    };
+}

src/intel/compiler/targets/gen9/compute/regions.zig

@@ -0,0 +1,49 @@
+const operand = @import("../../../ir/operand.zig");
+const program_ir = @import("../../../ir/program.zig");
+
+pub fn run(program: *program_ir.Program) void {
+    if (program.properties.regions_legalized)
+        return;
+
+    for (program.instructions.entries.items) |*entry| {
+        const inst = if (entry.*) |*value| value else continue;
+        switch (inst.operation) {
+            .load_buffer => |*op| legalizeSource(&op.byte_offset, inst.execution_size),
+            .store_buffer => |*op| {
+                legalizeSource(&op.byte_offset, inst.execution_size);
+                legalizeSource(&op.source, inst.execution_size);
+            },
+            .array_length => |*op| legalizeSource(&op.byte_offset, inst.execution_size),
+            .surface_read => |*op| legalizeSource(&op.address, inst.execution_size),
+            .surface_write => |*op| {
+                legalizeSource(&op.address, inst.execution_size);
+                legalizeSource(&op.data, inst.execution_size);
+            },
+            .move => |*op| legalizeSource(&op.source, inst.execution_size),
+            .binary => |*op| {
+                legalizeSource(&op.lhs, inst.execution_size);
+                legalizeSource(&op.rhs, inst.execution_size);
+            },
+            .math => |*op| {
+                legalizeSource(&op.lhs, inst.execution_size);
+                legalizeSource(&op.rhs, inst.execution_size);
+            },
+            .compare => |*op| {
+                legalizeSource(&op.lhs, inst.execution_size);
+                legalizeSource(&op.rhs, inst.execution_size);
+            },
+            else => {},
+        }
+    }
+
+    program.properties.regions_legalized = true;
+}
+
+fn legalizeSource(source: *operand.Source, execution_size: @import("../../../device.zig").ExecutionSize) void {
+    const byte_offset = source.region.byte_offset;
+    source.region = switch (source.register) {
+        .immediate => operand.Region.broadcast(),
+        else => operand.Region.contiguous(execution_size),
+    };
+    source.region.byte_offset = byte_offset;
+}

src/intel/compiler/targets/gen9/compute/resource_layout.zig

@@ -0,0 +1,141 @@
+const std = @import("std");
+const ids = @import("../../../ir/id.zig");
+const program_ir = @import("../../../ir/program.zig");
+
+pub const max_storage_buffers: usize = 4;
+
+pub const Error = std.mem.Allocator.Error || error{
+    TooManyStorageBuffers,
+};
+
+pub const Binding = struct {
+    set: u32,
+    binding: u32,
+    binding_table_index: u8,
+};
+
+const Candidate = struct {
+    resource: ids.StorageBufferId,
+    set: u32,
+    binding: u32,
+};
+
+pub const Layout = struct {
+    bindings: []Binding,
+    resource_indices: []?u8,
+
+    pub fn init(allocator: std.mem.Allocator, program: *const program_ir.Program) Error!Layout {
+        var candidates: std.ArrayList(Candidate) = .empty;
+        defer candidates.deinit(allocator);
+
+        for (program.storage_buffers.entries.items, 0..) |entry, index| {
+            const buffer = entry orelse continue;
+            try candidates.append(allocator, .{
+                .resource = ids.StorageBufferId.fromIndex(index),
+                .set = buffer.set,
+                .binding = buffer.binding,
+            });
+        }
+        std.mem.sort(Candidate, candidates.items, {}, lessThan);
+
+        var unique_count: usize = 0;
+        for (candidates.items, 0..) |candidate, index| {
+            if (index == 0 or candidate.set != candidates.items[index - 1].set or candidate.binding != candidates.items[index - 1].binding)
+                unique_count += 1;
+        }
+        if (unique_count > max_storage_buffers)
+            return Error.TooManyStorageBuffers;
+
+        const bindings = try allocator.alloc(Binding, unique_count);
+        errdefer allocator.free(bindings);
+        const resource_indices = try allocator.alloc(?u8, program.storage_buffers.entries.items.len);
+        errdefer allocator.free(resource_indices);
+        @memset(resource_indices, null);
+
+        var binding_index: usize = 0;
+        for (candidates.items, 0..) |candidate, index| {
+            if (index == 0 or candidate.set != candidates.items[index - 1].set or candidate.binding != candidates.items[index - 1].binding) {
+                bindings[binding_index] = .{
+                    .set = candidate.set,
+                    .binding = candidate.binding,
+                    .binding_table_index = @intCast(binding_index),
+                };
+                binding_index += 1;
+            }
+            resource_indices[candidate.resource.index()] = @intCast(binding_index - 1);
+        }
+        std.debug.assert(binding_index == bindings.len);
+
+        return .{
+            .bindings = bindings,
+            .resource_indices = resource_indices,
+        };
+    }
+
+    pub fn deinit(self: *Layout, allocator: std.mem.Allocator) void {
+        allocator.free(self.bindings);
+        allocator.free(self.resource_indices);
+        self.* = undefined;
+    }
+
+    pub fn bindingTableIndex(self: *const Layout, resource: ids.StorageBufferId) ?u8 {
+        if (resource.index() >= self.resource_indices.len)
+            return null;
+        return self.resource_indices[resource.index()];
+    }
+};
+
+fn lessThan(_: void, lhs: Candidate, rhs: Candidate) bool {
+    if (lhs.set != rhs.set)
+        return lhs.set < rhs.set;
+    if (lhs.binding != rhs.binding)
+        return lhs.binding < rhs.binding;
+    return lhs.resource.index() < rhs.resource.index();
+}
+
+test "[gen9] compute resource layout: assign stable binding-table indices" {
+    const device = @import("../../../device.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+
+    const third = try program.addStorageBuffer(.{ .set = 2, .binding = 7 });
+    const first = try program.addStorageBuffer(.{ .set = 0, .binding = 3 });
+    const alias = try program.addStorageBuffer(.{ .set = 0, .binding = 3 });
+    const second = try program.addStorageBuffer(.{ .set = 1, .binding = 0 });
+
+    var layout = try Layout.init(std.testing.allocator, &program);
+    defer layout.deinit(std.testing.allocator);
+
+    try std.testing.expectEqual(@as(usize, 3), layout.bindings.len);
+    try std.testing.expectEqual(Binding{ .set = 0, .binding = 3, .binding_table_index = 0 }, layout.bindings[0]);
+    try std.testing.expectEqual(Binding{ .set = 1, .binding = 0, .binding_table_index = 1 }, layout.bindings[1]);
+    try std.testing.expectEqual(Binding{ .set = 2, .binding = 7, .binding_table_index = 2 }, layout.bindings[2]);
+    try std.testing.expectEqual(@as(?u8, 0), layout.bindingTableIndex(first));
+    try std.testing.expectEqual(@as(?u8, 0), layout.bindingTableIndex(alias));
+    try std.testing.expectEqual(@as(?u8, 1), layout.bindingTableIndex(second));
+    try std.testing.expectEqual(@as(?u8, 2), layout.bindingTableIndex(third));
+}
+
+test "[gen9] compute resource layout: enforce advertised storage-buffer limit" {
+    const device = @import("../../../device.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+    for (0..max_storage_buffers + 1) |binding|
+        _ = try program.addStorageBuffer(.{ .set = 0, .binding = @intCast(binding) });
+
+    try std.testing.expectError(Error.TooManyStorageBuffers, Layout.init(std.testing.allocator, &program));
+}

src/intel/compiler/targets/gen9/compute/resource_lowering.zig

@@ -0,0 +1,129 @@
+const instruction = @import("../../../ir/instruction.zig");
+const program_ir = @import("../../../ir/program.zig");
+const validator = @import("../../../ir/validator.zig");
+const resource_layout = @import("resource_layout.zig");
+
+pub const Error = error{
+    InvalidProgram,
+    InvalidResourceLayout,
+};
+
+pub fn run(program: *program_ir.Program, layout: *const resource_layout.Layout) Error!void {
+    validator.validate(program) catch return Error.InvalidProgram;
+    if (program.properties.resources_lowered)
+        return;
+    if (layout.resource_indices.len != program.storage_buffers.entries.items.len)
+        return Error.InvalidResourceLayout;
+
+    for (program.blocks.entries.items) |block_entry| {
+        const block = block_entry orelse continue;
+        for (block.instructions.items) |instruction_id| {
+            const inst = program.instructions.get(instruction_id) orelse return Error.InvalidProgram;
+            const reference = bufferReference(inst.operation) orelse continue;
+            const resource = switch (reference) {
+                .logical => |value| value,
+                .binding_table => return Error.InvalidProgram,
+            };
+            const binding_table_index = layout.bindingTableIndex(resource) orelse return Error.InvalidResourceLayout;
+            if (binding_table_index >= layout.bindings.len)
+                return Error.InvalidResourceLayout;
+            const buffer = program.storage_buffers.get(resource) orelse return Error.InvalidProgram;
+            const binding = layout.bindings[binding_table_index];
+            if (binding.binding_table_index != binding_table_index or binding.set != buffer.set or binding.binding != buffer.binding)
+                return Error.InvalidResourceLayout;
+        }
+    }
+
+    for (program.blocks.entries.items) |block_entry| {
+        const block = block_entry orelse continue;
+        for (block.instructions.items) |instruction_id| {
+            const inst = program.instructions.getMut(instruction_id) orelse unreachable;
+            const reference = bufferReferenceMut(&inst.operation) orelse continue;
+            const resource = reference.logical;
+            const binding_table_index = layout.bindingTableIndex(resource).?;
+            reference.* = .{ .binding_table = binding_table_index };
+        }
+    }
+
+    program.properties.resources_lowered = true;
+    validator.validate(program) catch return Error.InvalidProgram;
+}
+
+fn bufferReference(operation: instruction.Operation) ?instruction.BufferReference {
+    return switch (operation) {
+        .load_buffer => |op| op.buffer,
+        .store_buffer => |op| op.buffer,
+        .array_length => |op| op.buffer,
+        else => null,
+    };
+}
+
+fn bufferReferenceMut(operation: *instruction.Operation) ?*instruction.BufferReference {
+    return switch (operation.*) {
+        .load_buffer => |*op| &op.buffer,
+        .store_buffer => |*op| &op.buffer,
+        .array_length => |*op| &op.buffer,
+        else => null,
+    };
+}
+
+test "[gen9] compute resource lowering: resolve logical buffers" {
+    const std = @import("std");
+    const Builder = @import("../../../ir/Builder.zig");
+    const device = @import("../../../device.zig");
+    const operand = @import("../../../ir/operand.zig");
+    const printer = @import("../../../ir/printer.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const value = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const buffer = try builder.addStorageBuffer(.{ .set = 1, .binding = 3, .name = "storage" });
+    const entry = try builder.addBlock("entry");
+    const store_id = try builder.appendInstruction(entry, .simd8, null, .{
+        .store_buffer = .{
+            .buffer = .{ .logical = buffer },
+            .byte_offset = .{
+                .register = .{ .immediate = .{ .u32 = 0 } },
+                .type = .u32,
+                .region = operand.Region.broadcast(),
+            },
+            .source = .{
+                .register = .{ .virtual = value },
+                .type = .u32,
+                .region = operand.Region.contiguous(.simd8),
+            },
+        },
+    });
+    try builder.setTerminator(entry, .end_thread);
+
+    var layout = try resource_layout.Layout.init(std.testing.allocator, &program);
+    defer layout.deinit(std.testing.allocator);
+    layout.bindings[0].binding = 4;
+    try std.testing.expectError(Error.InvalidResourceLayout, run(&program, &layout));
+    try std.testing.expect(!program.properties.resources_lowered);
+    try std.testing.expect(program.instructions.get(store_id).?.operation.store_buffer.buffer == .logical);
+    layout.bindings[0].binding = 3;
+
+    try run(&program, &layout);
+    try validator.validate(&program);
+
+    try std.testing.expect(program.properties.resources_lowered);
+    try std.testing.expectEqual(@as(u8, 0), program.instructions.get(store_id).?.operation.store_buffer.buffer.binding_table);
+    const text = try printer.allocPrint(std.testing.allocator, &program);
+    defer std.testing.allocator.free(text);
+    try std.testing.expect(std.mem.indexOf(u8, text, "store_buffer bti(0), 0:u32") != null);
+}

src/intel/compiler/targets/gen9/compute/system_values.zig

@@ -0,0 +1,95 @@
+const std = @import("std");
+
+const operand = @import("../../../ir/operand.zig");
+const program_ir = @import("../../../ir/program.zig");
+const validator = @import("../../../ir/validator.zig");
+
+pub const Error = error{InvalidProgram};
+
+pub fn run(program: *program_ir.Program) Error!void {
+    validator.validate(program) catch return Error.InvalidProgram;
+    if (program.properties.system_values_lowered)
+        return;
+
+    // Flint dispatch currently accepts only one invocation in one workgroup at
+    // base group zero, so every component of GlobalInvocationId is zero.
+    if (!std.mem.eql(u32, &program.workgroup_size, &.{ 1, 1, 1 }))
+        return;
+
+    for (program.instructions.entries.items) |*entry| {
+        const inst = if (entry.*) |*value| value else continue;
+        inst.operation = switch (inst.operation) {
+            .load_global_invocation_id => |op| .{ .move = .{
+                .destination = op.destination,
+                .source = zero(),
+            } },
+            else => inst.operation,
+        };
+    }
+
+    program.properties.system_values_lowered = true;
+    validator.validate(program) catch return Error.InvalidProgram;
+}
+
+fn zero() operand.Source {
+    return .{
+        .register = .{ .immediate = .{ .u32 = 0 } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    };
+}
+
+test "[gen9] system values: lower global invocation ID for single invocation" {
+    const Builder = @import("../../../ir/Builder.zig");
+    const device = @import("../../../device.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+
+    const destination = try builder.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const entry = try builder.addBlock("entry");
+    const load = try builder.appendInstruction(entry, .simd8, null, .{ .load_global_invocation_id = .{
+        .destination = .{ .register = .{ .virtual = destination }, .type = .u32 },
+        .component = 2,
+    } });
+    try builder.setTerminator(entry, .end_thread);
+    try builder.setEntryBlock(entry);
+
+    try run(&program);
+
+    try std.testing.expect(program.properties.system_values_lowered);
+    const move = program.instructions.get(load).?.operation.move;
+    try std.testing.expectEqual(@as(u32, 0), move.source.register.immediate.u32);
+}
+
+test "[gen9] system values: preserve IDs for unsupported workgroup sizes" {
+    const device = @import("../../../device.zig");
+
+    const device_info: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var program = program_ir.Program.init(std.testing.allocator, .{ 2, 1, 1 }, device_info, .simd8);
+    defer program.deinit();
+    const entry = try program.addBlock("entry");
+    try program.setTerminator(entry, .end_thread);
+    try program.setEntryBlock(entry);
+
+    try run(&program);
+    try std.testing.expect(!program.properties.system_values_lowered);
+}

src/intel/compiler/targets/gen9/flag_allocation.zig

@@ -0,0 +1,257 @@
+const std = @import("std");
+
+const ids = @import("../../ir/id.zig");
+const instruction = @import("../../ir/instruction.zig");
+const operand = @import("../../ir/operand.zig");
+const program_ir = @import("../../ir/program.zig");
+const pseudo = @import("../../ir/pseudo.zig");
+const validator = @import("validator.zig");
+
+pub const Error = std.mem.Allocator.Error || error{
+    InvalidProgram,
+    BlockParametersNotLowered,
+    ParallelCopiesNotLowered,
+    OutOfFlagRegisters,
+};
+
+const physical_flag_count = 2;
+
+pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!void {
+    if (!program.properties.block_parameters_lowered)
+        return Error.BlockParametersNotLowered;
+
+    if (!program.properties.parallel_copies_lowered)
+        return Error.ParallelCopiesNotLowered;
+
+    if (program.properties.flags_allocated)
+        return;
+
+    validator.validate(program) catch return Error.InvalidProgram;
+
+    const allocations = try allocator.alloc(?operand.PhysicalFlag, program.virtual_flags.entries.items.len);
+    defer allocator.free(allocations);
+    @memset(allocations, null);
+
+    var occupied: [physical_flag_count]bool = @splat(false);
+    try visitProgramFlags(program, allocations, &occupied, false);
+
+    for (allocations) |*allocation| {
+        const marker = allocation.* orelse continue;
+        if (marker.subregister != std.math.maxInt(u8))
+            return Error.InvalidProgram;
+
+        const subregister = std.mem.indexOfScalar(bool, &occupied, false) orelse return Error.OutOfFlagRegisters;
+
+        allocation.* = .{
+            .register = 0,
+            .subregister = @intCast(subregister),
+        };
+        occupied[subregister] = true;
+    }
+
+    try visitProgramFlags(program, allocations, &occupied, true);
+    program.properties.flags_allocated = true;
+    validator.validate(program) catch return Error.InvalidProgram;
+}
+
+fn visitProgramFlags(
+    program: *program_ir.Program,
+    allocations: []?operand.PhysicalFlag,
+    occupied: *[physical_flag_count]bool,
+    rewrite: bool,
+) Error!void {
+    for (program.instructions.entries.items, 0..) |entry, instruction_index| {
+        _ = entry orelse continue;
+        const inst = program.instructions.getMut(ids.InstructionId.fromIndex(instruction_index)) orelse
+            return Error.InvalidProgram;
+
+        if (inst.predicate) |*predicate|
+            try visitFlagRef(program, &predicate.flag, allocations, occupied, rewrite);
+
+        switch (inst.operation) {
+            .compare => |*compare| try visitFlagRef(program, &compare.destination, allocations, occupied, rewrite),
+            .parallel_copy => return Error.ParallelCopiesNotLowered,
+            else => {},
+        }
+    }
+
+    for (program.blocks.entries.items, 0..) |entry, block_index| {
+        _ = entry orelse continue;
+        const block = program.blocks.getMut(ids.BlockId.fromIndex(block_index)) orelse
+            return Error.InvalidProgram;
+        const terminator = if (block.terminator) |*value| value else return Error.InvalidProgram;
+
+        switch (terminator.*) {
+            .jump => |*edge| try visitEdge(program, edge, allocations, occupied, rewrite),
+            .conditional_branch => |*branch| {
+                try visitFlagRef(program, &branch.predicate.flag, allocations, occupied, rewrite);
+                try visitEdge(program, &branch.true_edge, allocations, occupied, rewrite);
+                try visitEdge(program, &branch.false_edge, allocations, occupied, rewrite);
+            },
+            .end_thread, .@"unreachable" => {},
+        }
+    }
+}
+
+fn visitEdge(
+    program: *const program_ir.Program,
+    edge: *instruction.Edge,
+    allocations: []?operand.PhysicalFlag,
+    occupied: *[physical_flag_count]bool,
+    rewrite: bool,
+) Error!void {
+    for (@constCast(edge.arguments)) |*argument| switch (argument.*) {
+        .source => {},
+        .predicate => |*value| try visitPredicateValue(program, value, allocations, occupied, rewrite),
+    };
+}
+
+fn visitPredicateValue(
+    program: *const program_ir.Program,
+    value: *pseudo.PredicateValue,
+    allocations: []?operand.PhysicalFlag,
+    occupied: *[physical_flag_count]bool,
+    rewrite: bool,
+) Error!void {
+    switch (value.*) {
+        .constant => {},
+        .dynamic => |*predicate| try visitFlagRef(program, &predicate.flag, allocations, occupied, rewrite),
+    }
+}
+
+fn visitFlagRef(
+    program: *const program_ir.Program,
+    flag: *operand.FlagRef,
+    allocations: []?operand.PhysicalFlag,
+    occupied: *[physical_flag_count]bool,
+    rewrite: bool,
+) Error!void {
+    switch (flag.*) {
+        .virtual => |virtual| {
+            if (!program.virtual_flags.isLive(virtual) or virtual.index() >= allocations.len)
+                return Error.InvalidProgram;
+
+            if (!rewrite) {
+                // Mark this virtual flag as referenced without assigning a physical
+                // slot until all pre-existing physical references are known.
+                if (allocations[virtual.index()] == null)
+                    allocations[virtual.index()] = .{ .register = 0, .subregister = std.math.maxInt(u8) };
+                return;
+            }
+
+            const physical = allocations[virtual.index()] orelse return Error.InvalidProgram;
+            if (physical.subregister >= physical_flag_count)
+                return Error.InvalidProgram;
+            flag.* = .{ .physical = physical };
+        },
+        .physical => |physical| {
+            if (physical.register != 0 or physical.subregister >= physical_flag_count)
+                return Error.InvalidProgram;
+            occupied[physical.subregister] = true;
+        },
+    }
+}
+
+const test_device = @import("../../device.zig").DeviceInfo{
+    .generation = .gen9,
+    .platform = .skylake,
+    .pci_device_id = 0x1912,
+    .grf_count = 128,
+};
+
+fn immediateU32(value: u32) operand.Source {
+    return .{
+        .register = .{ .immediate = .{ .u32 = value } },
+        .type = .u32,
+        .region = operand.Region.broadcast(),
+    };
+}
+
+fn markPrerequisites(program: *program_ir.Program) void {
+    program.properties.block_parameters_lowered = true;
+    program.properties.parallel_copies_lowered = true;
+}
+
+test "[gen9] flag allocation: rewrite compares and predicates" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    const first = try program.addVirtualFlag(.{ .name = "first" });
+    const second = try program.addVirtualFlag(.{ .name = "second" });
+    const scratch = try program.addVirtualRegister(.{
+        .size_bytes = 32,
+        .alignment_bytes = 32,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+    const entry = try program.addBlock("entry");
+    const taken = try program.addBlock("taken");
+    const not_taken = try program.addBlock("not_taken");
+
+    const first_compare = try program.appendInstruction(entry, .simd8, null, .{ .compare = .{
+        .opcode = .equal,
+        .destination = .{ .virtual = first },
+        .lhs = immediateU32(1),
+        .rhs = immediateU32(1),
+    } });
+    const predicated = try program.appendInstruction(entry, .simd8, .{
+        .flag = .{ .virtual = first },
+    }, .{ .compare = .{
+        .opcode = .not_equal,
+        .destination = .{ .virtual = second },
+        .lhs = .{
+            .register = .{ .virtual = scratch },
+            .type = .u32,
+            .region = operand.Region.contiguous(.simd8),
+        },
+        .rhs = immediateU32(0),
+    } });
+    try program.setTerminator(entry, .{ .conditional_branch = .{
+        .predicate = .{ .flag = .{ .virtual = second }, .inverse = true },
+        .true_edge = .{ .target = taken, .arguments = &.{} },
+        .false_edge = .{ .target = not_taken, .arguments = &.{} },
+    } });
+    try program.setTerminator(taken, .end_thread);
+    try program.setTerminator(not_taken, .end_thread);
+    markPrerequisites(&program);
+
+    try run(std.testing.allocator, &program);
+
+    try std.testing.expect(program.properties.flags_allocated);
+    try std.testing.expectEqual(@as(u8, 0), program.instructions.get(first_compare).?.operation.compare.destination.physical.subregister);
+    try std.testing.expectEqual(@as(u8, 0), program.instructions.get(predicated).?.predicate.?.flag.physical.subregister);
+    try std.testing.expectEqual(@as(u8, 1), program.instructions.get(predicated).?.operation.compare.destination.physical.subregister);
+    const branch = program.blocks.get(entry).?.terminator.?.conditional_branch;
+    try std.testing.expect(branch.predicate.inverse);
+    try std.testing.expectEqual(@as(u8, 1), branch.predicate.flag.physical.subregister);
+}
+
+test "[gen9] flag allocation: report exhaustion without rewriting" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+
+    const first = try program.addVirtualFlag(.{});
+    const second = try program.addVirtualFlag(.{});
+    const third = try program.addVirtualFlag(.{});
+    const entry = try program.addBlock("entry");
+
+    const first_compare = try program.appendInstruction(entry, .simd8, null, .{ .compare = .{
+        .opcode = .equal,
+        .destination = .{ .virtual = first },
+        .lhs = immediateU32(0),
+        .rhs = immediateU32(0),
+    } });
+    _ = try program.appendInstruction(entry, .simd8, .{ .flag = .{ .virtual = second } }, .{ .compare = .{
+        .opcode = .equal,
+        .destination = .{ .virtual = third },
+        .lhs = immediateU32(1),
+        .rhs = immediateU32(1),
+    } });
+    try program.setTerminator(entry, .end_thread);
+    markPrerequisites(&program);
+
+    try std.testing.expectError(Error.OutOfFlagRegisters, run(std.testing.allocator, &program));
+    try std.testing.expect(!program.properties.flags_allocated);
+    try std.testing.expectEqual(first, program.instructions.get(first_compare).?.operation.compare.destination.virtual);
+}

src/intel/compiler/targets/gen9/gen9.zig

@@ -0,0 +1,190 @@
+const std = @import("std");
+const shader_ir = @import("shader_ir").ir;
+const device = @import("../../device.zig");
+const common_ir = @import("../../lower/common_ir.zig");
+
+pub const compute = @import("compute/compute.zig");
+pub const compute_pipeline = @import("compute/pipeline.zig");
+pub const flag_allocation = @import("flag_allocation.zig");
+pub const register_allocation = @import("register_allocation.zig");
+pub const validator = @import("validator.zig");
+
+pub const Options = common_ir.Options;
+pub const ComputeArtifact = compute_pipeline.Artifact;
+pub const Error = compute_pipeline.Error;
+
+pub fn compileCompute(allocator: std.mem.Allocator, module: *shader_ir.module.Module, device_info: device.DeviceInfo, options: Options) Error!ComputeArtifact {
+    return compute_pipeline.compile(allocator, module, device_info, options);
+}
+
+test "[gen9] target: reject unsupported target configurations" {
+    var module = try shader_ir.parser.parseString(std.testing.allocator,
+        \\shader compute @main
+        \\{
+        \\    fn @main() -> void
+        \\    {
+        \\        .entry():
+        \\            return
+        \\    }
+        \\}
+    );
+    defer module.deinit();
+
+    const gen9_device: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var other_generation = gen9_device;
+    other_generation.generation = .gen11;
+    try std.testing.expectError(Error.UnsupportedGeneration, compileCompute(std.testing.allocator, &module, other_generation, .{}));
+
+    module.stage = .fragment;
+    try std.testing.expectError(Error.UnsupportedStage, compileCompute(std.testing.allocator, &module, gen9_device, .{}));
+    module.stage = .compute;
+
+    try std.testing.expectError(Error.UnsupportedDispatchWidth, compileCompute(std.testing.allocator, &module, gen9_device, .{ .dispatch_width = .simd16 }));
+
+    var wide_grf = gen9_device;
+    wide_grf.grf_size_bytes = 64;
+    try std.testing.expectError(Error.UnsupportedGrfSize, compileCompute(std.testing.allocator, &module, wide_grf, .{}));
+}
+
+test "[gen9] target: lower 256 KiB SSBO copy loop" {
+    const source =
+        \\shader compute @main
+        \\{
+        \\    @source: vec4[u32] = storage_buffer[set(0), binding(0)]
+        \\    @destination: vec4[u32] = storage_buffer[set(0), binding(1)]
+        \\    %zero: constant i32 = bits(0x0)
+        \\    %one: constant i32 = bits(0x1)
+        \\    %stride: constant i32 = bits(0x10)
+        \\    %element_count: constant i32 = bits(0x4000)
+        \\    fn @main() -> void
+        \\    {
+        \\        .entry():
+        \\            branch .header(%zero)
+        \\        .header(%index: i32):
+        \\            %in_bounds: bool = cmp_signed_less %index, %element_count
+        \\            conditional_branch %in_bounds, .body(), .exit()
+        \\        .body():
+        \\            %signed_offset: i32 = integer_multiply %index, %stride
+        \\            %offset: u32 = bitcast %signed_offset
+        \\            %value: vec4[u32] = load_buffer @source, %offset
+        \\            store_buffer @destination, %offset, %value
+        \\            branch .continue()
+        \\        .continue():
+        \\            %next: i32 = integer_add %index, %one
+        \\            branch .header(%next)
+        \\        .exit():
+        \\            return
+        \\    }
+        \\}
+    ;
+
+    var module = try shader_ir.parser.parseString(std.testing.allocator, source);
+    defer module.deinit();
+    module.execution_modes.workgroup_size = .{ 1, 1, 1 };
+
+    const gen9_device: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var artifact = try compileCompute(std.testing.allocator, &module, gen9_device, .{});
+    defer artifact.deinit(std.testing.allocator);
+    try std.testing.expect(artifact.kernel != null);
+    const program = &artifact.program;
+    const resources = &artifact.resources;
+
+    try std.testing.expect(program.properties.common_ir_lowered);
+    try std.testing.expect(program.properties.compute_abi_lowered);
+    try std.testing.expectEqual(@as(u16, 1), program.program_data.payload_grf_count);
+    try std.testing.expectEqual(@as(u16, 0), program.payload.header_grf.?.number);
+    try std.testing.expect(program.properties.block_parameters_lowered);
+    try std.testing.expect(program.properties.parallel_copies_lowered);
+    try std.testing.expect(program.properties.flags_allocated);
+    try std.testing.expect(program.properties.registers_allocated);
+
+    try std.testing.expect(program.properties.resources_lowered);
+    try std.testing.expect(program.properties.messages_lowered);
+    try std.testing.expect(program.properties.message_addresses_lowered);
+    try std.testing.expect(program.properties.message_payloads_lowered);
+    try std.testing.expectEqual(@as(usize, 2), resources.bindings.len);
+    try std.testing.expectEqual(compute.resource_layout.Binding{
+        .set = 0,
+        .binding = 0,
+        .binding_table_index = 0,
+    }, resources.bindings[0]);
+    try std.testing.expectEqual(compute.resource_layout.Binding{
+        .set = 0,
+        .binding = 1,
+        .binding_table_index = 1,
+    }, resources.bindings[1]);
+
+    var load_count: usize = 0;
+    var store_count: usize = 0;
+    for (program.instructions.entries.items) |instruction_entry| {
+        const inst = instruction_entry orelse continue;
+        switch (inst.operation) {
+            .surface_message => |operation| switch (operation.kind) {
+                .read => {
+                    try std.testing.expectEqual(@as(u8, 0), operation.binding_table);
+                    try std.testing.expectEqual(@as(u8, 1), operation.payload.register_count);
+                    try std.testing.expect(operation.response != null);
+                    load_count += 1;
+                },
+                .write => {
+                    try std.testing.expectEqual(@as(u8, 1), operation.binding_table);
+                    try std.testing.expectEqual(@as(u8, 2), operation.payload.register_count);
+                    try std.testing.expect(operation.response == null);
+                    store_count += 1;
+                },
+            },
+            .parallel_copy => return error.UnloweredParallelCopy,
+            else => {},
+        }
+    }
+
+    try std.testing.expectEqual(@as(usize, 4), load_count);
+    try std.testing.expectEqual(@as(usize, 4), store_count);
+}
+
+test "[gen9] target: encode runtime array length" {
+    var module = try shader_ir.parser.parseString(std.testing.allocator,
+        \\shader compute @main
+        \\{
+        \\    @storage: runtime_array[u32] = storage_buffer[set(0), binding(0)]
+        \\    %offset: constant u32 = 16
+        \\    fn @main() -> void
+        \\    {
+        \\        .entry():
+        \\            %length: u32 = array_length @storage, %offset, stride 4
+        \\            return
+        \\    }
+        \\}
+    );
+    defer module.deinit();
+    module.execution_modes.workgroup_size = .{ 1, 1, 1 };
+
+    const gen9_device: device.DeviceInfo = .{
+        .generation = .gen9,
+        .platform = .skylake,
+        .pci_device_id = 0x1912,
+        .grf_count = 128,
+    };
+    var artifact = try compileCompute(std.testing.allocator, &module, gen9_device, .{});
+    defer artifact.deinit(std.testing.allocator);
+
+    if (artifact.kernel == null) {
+        const encoded = try compute.kernel_encoder.encode(std.testing.allocator, &artifact.program);
+        std.testing.allocator.free(encoded);
+        return error.TestExpectedEncodedKernel;
+    }
+    for (artifact.program.instructions.entries.items) |entry| {
+        const inst = entry orelse continue;
+        try std.testing.expect(inst.operation != .array_length);
+    }
+}

src/intel/compiler/targets/gen9/register_allocation.zig

@@ -0,0 +1,285 @@
+const std = @import("std");
+
+const ids = @import("../../ir/id.zig");
+const instruction = @import("../../ir/instruction.zig");
+const operand = @import("../../ir/operand.zig");
+const program_ir = @import("../../ir/program.zig");
+
+pub const Error = std.mem.Allocator.Error || error{
+    BlockParametersNotLowered,
+    ParallelCopiesNotLowered,
+    InvalidProgram,
+    OutOfRegisters,
+};
+
+pub fn run(allocator: std.mem.Allocator, program: *program_ir.Program) Error!void {
+    if (!program.properties.block_parameters_lowered)
+        return Error.BlockParametersNotLowered;
+    if (!program.properties.parallel_copies_lowered)
+        return Error.ParallelCopiesNotLowered;
+    if (program.properties.registers_allocated)
+        return;
+
+    const grf_size = program.device_info.grf_size_bytes;
+    if (grf_size == 0)
+        return Error.InvalidProgram;
+
+    const allocations = try allocator.alloc(?operand.PhysicalGrf, program.virtual_registers.entries.items.len);
+    defer allocator.free(allocations);
+    @memset(allocations, null);
+
+    var next_byte: usize = @as(usize, program.program_data.payload_grf_count) * grf_size;
+    next_byte = try reserveExistingPhysicalRegisters(program, next_byte, grf_size);
+    const capacity: usize = @as(usize, program.device_info.grf_count) * grf_size;
+
+    for (program.virtual_registers.entries.items, 0..) |entry, index| {
+        const register = entry orelse continue;
+        const start = std.mem.alignForward(usize, next_byte, register.alignment_bytes);
+        const end = std.math.add(usize, start, register.size_bytes) catch return Error.OutOfRegisters;
+        if (end > capacity)
+            return Error.OutOfRegisters;
+
+        allocations[index] = .{
+            .number = @intCast(start / grf_size),
+            .byte_offset = @intCast(start % grf_size),
+        };
+        next_byte = end;
+    }
+
+    try rewriteProgram(program, allocations);
+    program.program_data.total_grf_count = @intCast(std.math.divCeil(usize, next_byte, grf_size) catch return Error.InvalidProgram);
+    program.properties.registers_allocated = true;
+}
+
+fn reserveExistingPhysicalRegisters(program: *const program_ir.Program, initial: usize, grf_size: usize) Error!usize {
+    var next_byte = initial;
+    if (program.payload.header_grf) |header|
+        reservePhysical(&next_byte, header, grf_size);
+
+    for (program.instructions.entries.items) |entry| {
+        const inst = entry orelse continue;
+        switch (inst.operation) {
+            .load_global_invocation_id => |op| reserveRegister(&next_byte, op.destination.register, grf_size),
+            .load_buffer => |op| {
+                reserveRegister(&next_byte, op.destination.register, grf_size);
+                reserveRegister(&next_byte, op.byte_offset.register, grf_size);
+            },
+            .store_buffer => |op| {
+                reserveRegister(&next_byte, op.byte_offset.register, grf_size);
+                reserveRegister(&next_byte, op.source.register, grf_size);
+            },
+            .array_length => |op| {
+                reserveRegister(&next_byte, op.destination.register, grf_size);
+                reserveRegister(&next_byte, op.byte_offset.register, grf_size);
+            },
+            .surface_read => |op| {
+                reserveRegister(&next_byte, op.destination.register, grf_size);
+                reserveRegister(&next_byte, op.address.register, grf_size);
+            },
+            .surface_write => |op| {
+                reserveRegister(&next_byte, op.address.register, grf_size);
+                reserveRegister(&next_byte, op.data.register, grf_size);
+            },
+            .surface_message => |op| {
+                reserveRegister(&next_byte, op.payload.base, grf_size);
+                if (op.response) |response|
+                    reserveRegister(&next_byte, response.base, grf_size);
+            },
+            .move => |op| {
+                reserveRegister(&next_byte, op.destination.register, grf_size);
+                reserveRegister(&next_byte, op.source.register, grf_size);
+            },
+            .binary => |op| {
+                reserveRegister(&next_byte, op.destination.register, grf_size);
+                reserveRegister(&next_byte, op.lhs.register, grf_size);
+                reserveRegister(&next_byte, op.rhs.register, grf_size);
+            },
+            .compare => |op| {
+                reserveRegister(&next_byte, op.lhs.register, grf_size);
+                reserveRegister(&next_byte, op.rhs.register, grf_size);
+            },
+            .math => |op| {
+                reserveRegister(&next_byte, op.destination.register, grf_size);
+                reserveRegister(&next_byte, op.lhs.register, grf_size);
+                reserveRegister(&next_byte, op.rhs.register, grf_size);
+            },
+            .parallel_copy => return Error.ParallelCopiesNotLowered,
+        }
+    }
+    return next_byte;
+}
+
+fn reserveRegister(next_byte: *usize, register: operand.RegisterRef, grf_size: usize) void {
+    switch (register) {
+        .physical_grf => |physical| reservePhysical(next_byte, physical, grf_size),
+        else => {},
+    }
+}
+
+fn reservePhysical(next_byte: *usize, physical: operand.PhysicalGrf, grf_size: usize) void {
+    const end = (@as(usize, physical.number) + 1) * grf_size;
+    next_byte.* = @max(next_byte.*, end);
+}
+
+fn rewriteProgram(program: *program_ir.Program, allocations: []const ?operand.PhysicalGrf) Error!void {
+    for (program.instructions.entries.items) |*entry| {
+        const inst = if (entry.*) |*value| value else continue;
+        switch (inst.operation) {
+            .load_global_invocation_id => |*op| try rewriteDestination(program, &op.destination, allocations),
+            .load_buffer => |*op| {
+                try rewriteDestination(program, &op.destination, allocations);
+                try rewriteSource(program, &op.byte_offset, allocations);
+            },
+            .store_buffer => |*op| {
+                try rewriteSource(program, &op.byte_offset, allocations);
+                try rewriteSource(program, &op.source, allocations);
+            },
+            .array_length => |*op| {
+                try rewriteDestination(program, &op.destination, allocations);
+                try rewriteSource(program, &op.byte_offset, allocations);
+            },
+            .surface_read => |*op| {
+                try rewriteDestination(program, &op.destination, allocations);
+                try rewriteSource(program, &op.address, allocations);
+            },
+            .surface_write => |*op| {
+                try rewriteSource(program, &op.address, allocations);
+                try rewriteSource(program, &op.data, allocations);
+            },
+            .surface_message => |*op| {
+                try rewriteRegister(program, &op.payload.base, allocations);
+                if (op.response) |*response|
+                    try rewriteRegister(program, &response.base, allocations);
+            },
+            .move => |*op| {
+                try rewriteDestination(program, &op.destination, allocations);
+                try rewriteSource(program, &op.source, allocations);
+            },
+            .binary => |*op| {
+                try rewriteDestination(program, &op.destination, allocations);
+                try rewriteSource(program, &op.lhs, allocations);
+                try rewriteSource(program, &op.rhs, allocations);
+            },
+            .compare => |*op| {
+                try rewriteSource(program, &op.lhs, allocations);
+                try rewriteSource(program, &op.rhs, allocations);
+            },
+            .math => |*op| {
+                try rewriteDestination(program, &op.destination, allocations);
+                try rewriteSource(program, &op.lhs, allocations);
+                try rewriteSource(program, &op.rhs, allocations);
+            },
+            .parallel_copy => return Error.ParallelCopiesNotLowered,
+        }
+    }
+
+    for (program.blocks.entries.items) |*entry| {
+        const block = if (entry.*) |*value| value else continue;
+        if (block.parameters.items.len != 0)
+            return Error.BlockParametersNotLowered;
+        const terminator = if (block.terminator) |*value| value else return Error.InvalidProgram;
+        switch (terminator.*) {
+            .jump => |*edge| try rewriteEdge(program, edge, allocations),
+            .conditional_branch => |*branch| {
+                try rewriteEdge(program, &branch.true_edge, allocations);
+                try rewriteEdge(program, &branch.false_edge, allocations);
+            },
+            .end_thread, .@"unreachable" => {},
+        }
+    }
+}
+
+fn rewriteEdge(program: *const program_ir.Program, edge: *instruction.Edge, allocations: []const ?operand.PhysicalGrf) Error!void {
+    for (@constCast(edge.arguments)) |*argument| switch (argument.*) {
+        .source => |*edge_source| try rewriteSource(program, edge_source, allocations),
+        .predicate => {},
+    };
+}
+
+fn rewriteSource(program: *const program_ir.Program, value: *operand.Source, allocations: []const ?operand.PhysicalGrf) Error!void {
+    try rewriteRegister(program, &value.register, allocations);
+}
+
+fn rewriteDestination(program: *const program_ir.Program, destination: *operand.Destination, allocations: []const ?operand.PhysicalGrf) Error!void {
+    try rewriteRegister(program, &destination.register, allocations);
+}
+
+fn rewriteRegister(program: *const program_ir.Program, register: *operand.RegisterRef, allocations: []const ?operand.PhysicalGrf) Error!void {
+    const virtual = switch (register.*) {
+        .virtual => |value| value,
+        else => return,
+    };
+    if (!program.virtual_registers.isLive(virtual) or virtual.index() >= allocations.len)
+        return Error.InvalidProgram;
+    const physical = allocations[virtual.index()] orelse return Error.InvalidProgram;
+    register.* = .{ .physical_grf = physical };
+}
+
+const test_device = @import("../../device.zig").DeviceInfo{
+    .generation = .gen9,
+    .platform = .skylake,
+    .pci_device_id = 0x1912,
+    .grf_count = 128,
+};
+
+fn addRegister(program: *program_ir.Program, size: u32, alignment: u16) !ids.VirtualRegisterId {
+    return program.addVirtualRegister(.{
+        .size_bytes = size,
+        .alignment_bytes = alignment,
+        .element_type = .u32,
+        .lane_count = 8,
+        .class = .temporary,
+    });
+}
+
+fn source(register: ids.VirtualRegisterId) operand.Source {
+    return .{
+        .register = .{ .virtual = register },
+        .type = .u32,
+        .region = operand.Region.contiguous(.simd8),
+    };
+}
+
+fn markPrerequisites(program: *program_ir.Program) void {
+    program.properties.block_parameters_lowered = true;
+    program.properties.parallel_copies_lowered = true;
+}
+
+test "[gen9] register allocation: assign non-overlapping physical GRFs" {
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, test_device, .simd8);
+    defer program.deinit();
+    program.program_data.payload_grf_count = 1;
+
+    const first = try addRegister(&program, 32, 32);
+    const second = try addRegister(&program, 64, 32);
+    const entry = try program.addBlock("entry");
+    const move = try program.appendInstruction(entry, .simd8, null, .{ .move = .{
+        .destination = .{ .register = .{ .virtual = second }, .type = .u32 },
+        .source = source(first),
+    } });
+    try program.setTerminator(entry, .end_thread);
+    markPrerequisites(&program);
+
+    try run(std.testing.allocator, &program);
+
+    const operation = program.instructions.get(move).?.operation.move;
+    try std.testing.expectEqual(operand.PhysicalGrf{ .number = 1 }, operation.source.register.physical_grf);
+    try std.testing.expectEqual(operand.PhysicalGrf{ .number = 2 }, operation.destination.register.physical_grf);
+    try std.testing.expectEqual(@as(u16, 4), program.program_data.total_grf_count);
+    try std.testing.expect(program.properties.registers_allocated);
+}
+
+test "[gen9] register allocation: report GRF exhaustion" {
+    var limited_device = test_device;
+    limited_device.grf_count = 2;
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, limited_device, .simd8);
+    defer program.deinit();
+
+    _ = try addRegister(&program, 96, 32);
+    const entry = try program.addBlock("entry");
+    try program.setTerminator(entry, .end_thread);
+    markPrerequisites(&program);
+
+    try std.testing.expectError(Error.OutOfRegisters, run(std.testing.allocator, &program));
+    try std.testing.expect(!program.properties.registers_allocated);
+}

src/intel/compiler/targets/gen9/validator.zig

@@ -0,0 +1,209 @@
+const std = @import("std");
+const compute = @import("compute/compute.zig");
+const shared = @import("../../ir/validator.zig");
+const instruction = @import("../../ir/instruction.zig");
+const operand = @import("../../ir/operand.zig");
+const program_ir = @import("../../ir/program.zig");
+
+pub const Error = shared.Error || compute.Error || error{
+    UnsupportedGeneration,
+    UnsupportedDispatchWidth,
+    UnsupportedGrfSize,
+    UnsupportedExecutionSize,
+    UnsupportedDataType,
+    InvalidPhysicalFlag,
+    InvalidBindingTableIndex,
+    InvalidPayloadLayout,
+};
+
+pub fn validate(program: *const program_ir.Program) Error!void {
+    try shared.validate(program);
+
+    if (program.device_info.generation != .gen9)
+        return Error.UnsupportedGeneration;
+    try compute.validateWorkgroupSize(program.workgroup_size);
+    if (program.dispatch_width != .simd8 or !program.device_info.supportsDispatch(.simd8))
+        return Error.UnsupportedDispatchWidth;
+    if (program.device_info.grf_size_bytes != 32)
+        return Error.UnsupportedGrfSize;
+
+    for (program.blocks.entries.items) |block_entry| {
+        const block = block_entry orelse continue;
+        for (block.instructions.items) |instruction_id| {
+            const inst = program.instructions.get(instruction_id) orelse return Error.InvalidInstruction;
+            switch (inst.execution_size) {
+                .simd1, .simd8 => {},
+                else => return Error.UnsupportedExecutionSize,
+            }
+            try validateInstruction(inst.*);
+        }
+        try validateTerminator(block.terminator.?);
+    }
+
+    for (program.virtual_registers.entries.items) |entry| {
+        const register = entry orelse continue;
+        if (!register.element_type.isInitialTargetType())
+            return Error.UnsupportedDataType;
+    }
+
+    try validatePayload(program);
+}
+
+fn validateInstruction(inst: instruction.Instruction) Error!void {
+    if (inst.predicate) |predicate|
+        try validateFlag(predicate.flag);
+    switch (inst.operation) {
+        .load_global_invocation_id => |op| try validateDestination(op.destination),
+        .load_buffer => |op| {
+            try validateBufferReference(op.buffer);
+            try validateDestination(op.destination);
+            try validateSource(op.byte_offset);
+        },
+        .store_buffer => |op| {
+            try validateBufferReference(op.buffer);
+            try validateSource(op.byte_offset);
+            try validateSource(op.source);
+        },
+        .array_length => |op| {
+            try validateBufferReference(op.buffer);
+            try validateDestination(op.destination);
+            try validateSource(op.byte_offset);
+        },
+        .surface_read => |op| {
+            try validateBindingTableIndex(op.binding_table);
+            try validateDestination(op.destination);
+            try validateSource(op.address);
+        },
+        .surface_write => |op| {
+            try validateBindingTableIndex(op.binding_table);
+            try validateSource(op.address);
+            try validateSource(op.data);
+        },
+        .surface_message => |op| try validateBindingTableIndex(op.binding_table),
+        .move => |op| {
+            try validateDestination(op.destination);
+            try validateSource(op.source);
+        },
+        .binary => |op| {
+            try validateDestination(op.destination);
+            try validateSource(op.lhs);
+            try validateSource(op.rhs);
+        },
+        .compare => |op| {
+            try validateFlag(op.destination);
+            try validateSource(op.lhs);
+            try validateSource(op.rhs);
+        },
+        .math => |op| {
+            try validateDestination(op.destination);
+            try validateSource(op.lhs);
+            try validateSource(op.rhs);
+        },
+        .parallel_copy => |copy| {
+            for (copy.register_copies) |item| {
+                try validateDestination(item.destination);
+                try validateSource(item.source);
+            }
+            for (copy.flag_copies) |item| switch (item.source) {
+                .constant => {},
+                .dynamic => |predicate| try validateFlag(predicate.flag),
+            };
+        },
+    }
+}
+
+fn validateBufferReference(reference: instruction.BufferReference) Error!void {
+    switch (reference) {
+        .logical => {},
+        .binding_table => |index| try validateBindingTableIndex(index),
+    }
+}
+
+fn validateBindingTableIndex(index: u8) Error!void {
+    if (index >= compute.resource_layout.max_storage_buffers)
+        return Error.InvalidBindingTableIndex;
+}
+
+fn validateSource(source: operand.Source) Error!void {
+    try validateType(source.type);
+    switch (source.register) {
+        .immediate => |immediate| try validateImmediate(immediate),
+        else => {},
+    }
+}
+
+fn validateDestination(destination: operand.Destination) Error!void {
+    try validateType(destination.type);
+}
+
+fn validateType(data_type: operand.DataType) Error!void {
+    if (!data_type.isInitialTargetType())
+        return Error.UnsupportedDataType;
+}
+
+fn validateImmediate(immediate: operand.Immediate) Error!void {
+    switch (immediate) {
+        .u32, .i32, .f32 => {},
+    }
+}
+
+fn validateTerminator(terminator: instruction.Terminator) Error!void {
+    switch (terminator) {
+        .conditional_branch => |branch| {
+            try validateFlag(branch.predicate.flag);
+            try validateEdge(branch.true_edge);
+            try validateEdge(branch.false_edge);
+        },
+        .jump => |edge| try validateEdge(edge),
+        else => {},
+    }
+}
+
+fn validateEdge(edge: instruction.Edge) Error!void {
+    for (edge.arguments) |argument| switch (argument) {
+        .source => {},
+        .predicate => |predicate_value| switch (predicate_value) {
+            .constant => {},
+            .dynamic => |predicate| try validateFlag(predicate.flag),
+        },
+    };
+}
+
+fn validateFlag(flag: operand.FlagRef) Error!void {
+    switch (flag) {
+        .virtual => {},
+        .physical => |physical| if (physical.register != 0 or physical.subregister > 1)
+            return Error.InvalidPhysicalFlag,
+    }
+}
+
+fn validatePayload(program: *const program_ir.Program) Error!void {
+    if (program.payload.header_grf) |header| {
+        if (header.number != 0 or header.byte_offset != 0)
+            return Error.InvalidPayloadLayout;
+    }
+}
+
+test "[gen9] validator: layer target legality over shared structural validation" {
+    const Builder = @import("../../ir/Builder.zig");
+    const device = @import("../../device.zig");
+
+    const gen11_device: device.DeviceInfo = .{
+        .generation = .gen11,
+        .platform = .ice_lake,
+        .pci_device_id = 0x8a52,
+        .grf_count = 128,
+        .supports_simd16 = true,
+    };
+    var program = program_ir.Program.init(std.testing.allocator, .{ 1, 1, 1 }, gen11_device, .simd16);
+    defer program.deinit();
+    var builder = Builder.init(&program);
+    const entry = try builder.addBlock("entry");
+    try builder.setTerminator(entry, .end_thread);
+
+    try shared.validate(&program);
+    try std.testing.expectError(Error.UnsupportedGeneration, validate(&program));
+
+    program.device_info.generation = .gen9;
+    try std.testing.expectError(Error.UnsupportedDispatchWidth, validate(&program));
+}

src/intel/compiler/targets/targets.zig

@@ -0,0 +1,31 @@
+const std = @import("std");
+const shader_ir = @import("shader_ir").ir;
+const device = @import("../device.zig");
+const program_ir = @import("../ir/program.zig");
+const common_ir = @import("../lower/common_ir.zig");
+
+pub const gen9 = @import("gen9/gen9.zig");
+
+pub const ComputeArtifact = gen9.ComputeArtifact;
+pub const ComputeResourceLayout = gen9.compute.ResourceLayout;
+pub const Error = gen9.Error || error{UnsupportedGeneration};
+pub const ValidationError = gen9.validator.Error || error{UnsupportedGeneration};
+
+pub fn compileCompute(
+    allocator: std.mem.Allocator,
+    module: *shader_ir.module.Module,
+    device_info: device.DeviceInfo,
+    options: common_ir.Options,
+) Error!ComputeArtifact {
+    return switch (device_info.generation) {
+        .gen9 => gen9.compileCompute(allocator, module, device_info, options),
+        .gen10, .gen11 => Error.UnsupportedGeneration,
+    };
+}
+
+pub fn validate(program: *const program_ir.Program) ValidationError!void {
+    return switch (program.device_info.generation) {
+        .gen9 => gen9.validator.validate(program),
+        .gen10, .gen11 => ValidationError.UnsupportedGeneration,
+    };
+}

src/intel/copy_commands.zig

@@ -9,6 +9,7 @@ const FlintCommandBuffer = @import("FlintCommandBuffer.zig");
 const MemoryRange = @import("MemoryRange.zig");
 
 pub fn emitLinearCopy(cmd: *FlintCommandBuffer, src: MemoryRange, dst: MemoryRange) VkError!void {
+    try cmd.requireEngine(.blitter);
     if (src.size != dst.size) return VkError.ValidationFailed;
 
     var copied: vk.DeviceSize = 0;

src/intel/i915/i915.zig

@@ -7,13 +7,17 @@ pub const gem_close = 0x09;
 
 pub const mmap_offset_wb = 2;
 pub const gem_domain_cpu = 0x00000001;
+pub const gem_domain_render = 0x00000004;
+pub const gem_domain_instruction = 0x00000010;
 pub const gem_domain_gtt = 0x00000040;
+pub const exec_render = 1 << 0;
 pub const exec_blt = 3 << 0;
 pub const exec_fence_array: u64 = 1 << 19;
 pub const exec_fence_wait: u32 = 1 << 0;
 pub const exec_fence_signal: u32 = 1 << 1;
 pub const exec_object_write = 1 << 2;
 pub const mi_flush_dw: u32 = (0x26 << 23) | 3;
+pub const mi_batch_buffer_end: u32 = 0x05000000;
 
 pub const GemCreate = extern struct {
     size: u64,

src/intel/i915/kmd.zig

@@ -7,6 +7,11 @@ const common_kmd = @import("../kmd.zig");
 
 const VkError = base.VkError;
 
+const RelocationGroup = struct {
+    source_handle: u32,
+    entries: std.ArrayList(_i915.RelocationEntry) = .empty,
+};
+
 const Mapping = struct {
     bytes: []align(std.heap.page_size_min) u8,
 
@@ -54,8 +59,19 @@ pub const Device = struct {
         return memory;
     }
 
-    pub fn submitBatch(self: *Device, io: std.Io, allocator: std.mem.Allocator, commands: []const u32, relocations: []const common_kmd.Relocation, syncs: []const common_kmd.SyncDependency) VkError!void {
+    pub fn submitBatch(
-        const trailer_words = 6;
+        self: *Device,
+        io: std.Io,
+        allocator: std.mem.Allocator,
+        engine: common_kmd.Engine,
+        commands: []const u32,
+        relocations: []const common_kmd.Relocation,
+        syncs: []const common_kmd.SyncDependency,
+    ) VkError!void {
+        const trailer_words: usize = switch (engine) {
+            .blitter => 6,
+            .render => if (commands.len % 2 == 0) 2 else 1,
+        };
         const batch_size = (commands.len + trailer_words) * @sizeOf(u32);
         var batch = try self.allocateMemory(io, batch_size);
         defer batch.deinit(self, io);
@@ -64,65 +80,93 @@ pub const Device = struct {
             const batch_map = try batch.map(self, io, 0, batch_size);
             const batch_words = std.mem.bytesAsSlice(u32, batch_map);
             @memcpy(batch_words[0..commands.len], commands);
-            batch_words[commands.len + 0] = _i915.mi_flush_dw;
+            @memset(batch_words[commands.len..], 0);
-            batch_words[commands.len + 1] = 0;
+            switch (engine) {
-            batch_words[commands.len + 2] = 0;
+                .blitter => {
-            batch_words[commands.len + 3] = 0;
+                    batch_words[commands.len] = _i915.mi_flush_dw;
-            batch_words[commands.len + 4] = 0;
+                    batch_words[commands.len + 5] = _i915.mi_batch_buffer_end;
-            batch_words[commands.len + 5] = 0x05000000;
+                },
+                .render => batch_words[commands.len] = _i915.mi_batch_buffer_end,
+            }
             batch.unmap();
         }
         try batch.flushRange(self, io, 0, batch_size);
 
-        var objects = std.ArrayList(_i915.ExecObject2).empty;
-        defer objects.deinit(allocator);
-
         var object_handles = std.ArrayList(u32).empty;
         defer object_handles.deinit(allocator);
-
         for (relocations) |relocation| {
-            if (std.mem.indexOfScalar(u32, object_handles.items, relocation.target_handle) == null) {
+            if (relocation.source_handle) |source| {
-                object_handles.append(allocator, relocation.target_handle) catch return VkError.OutOfHostMemory;
+                if (std.mem.indexOfScalar(u32, object_handles.items, source) == null)
-                objects.append(allocator, .{
+                    object_handles.append(allocator, source) catch return VkError.OutOfHostMemory;
-                    .handle = relocation.target_handle,
-                    .relocation_count = 0,
-                    .relocs_ptr = 0,
-                    .alignment = 0,
-                    .offset = 0,
-                    .flags = if (relocation.write) _i915.exec_object_write else 0,
-                    .rsvd1 = 0,
-                    .rsvd2 = 0,
-                }) catch return VkError.OutOfHostMemory;
-            } else if (relocation.write) {
-                const index = std.mem.indexOfScalar(u32, object_handles.items, relocation.target_handle).?;
-                objects.items[index].flags |= _i915.exec_object_write;
             }
+            if (std.mem.indexOfScalar(u32, object_handles.items, relocation.target_handle) == null)
+                object_handles.append(allocator, relocation.target_handle) catch return VkError.OutOfHostMemory;
         }
+        if (std.mem.indexOfScalar(u32, object_handles.items, batch.handle) == null)
+            object_handles.append(allocator, batch.handle) catch return VkError.OutOfHostMemory;
 
-        var i915_relocations = std.ArrayList(_i915.RelocationEntry).empty;
+        var groups = std.ArrayList(RelocationGroup).empty;
-        defer i915_relocations.deinit(allocator);
+        defer {
-
+            for (groups.items) |*group| group.entries.deinit(allocator);
+            groups.deinit(allocator);
+        }
         for (relocations) |relocation| {
-            i915_relocations.append(allocator, .{
+            const source = relocation.source_handle orelse batch.handle;
+            var group_index = std.mem.indexOfScalar(u32, object_handles.items, source) orelse return VkError.DeviceLost;
+            for (groups.items, 0..) |group, index| {
+                if (group.source_handle == source) {
+                    group_index = index;
+                    break;
+                }
+            } else {
+                groups.append(allocator, .{ .source_handle = source }) catch return VkError.OutOfHostMemory;
+                group_index = groups.items.len - 1;
+            }
+
+            const domain: u32 = switch (relocation.domain) {
+                .none => 0,
+                .render => _i915.gem_domain_render,
+                .instruction => _i915.gem_domain_instruction,
+            };
+            groups.items[group_index].entries.append(allocator, .{
                 .target_handle = relocation.target_handle,
                 .delta = relocation.delta,
                 .offset = relocation.offset,
                 .presumed_offset = 0,
-                .read_domains = 0,
+                .read_domains = if (relocation.read) domain else 0,
-                .write_domain = 0,
+                .write_domain = if (relocation.write) domain else 0,
             }) catch return VkError.OutOfHostMemory;
         }
 
-        objects.append(allocator, .{
+        var objects = std.ArrayList(_i915.ExecObject2).empty;
-            .handle = batch.handle,
+        defer objects.deinit(allocator);
-            .relocation_count = @intCast(i915_relocations.items.len),
+        for (object_handles.items) |handle| {
-            .relocs_ptr = @intFromPtr(i915_relocations.items.ptr),
+            var flags: u64 = 0;
-            .alignment = 0,
+            for (relocations) |relocation| {
-            .offset = 0,
+                if (relocation.target_handle == handle and relocation.write)
-            .flags = 0,
+                    flags |= _i915.exec_object_write;
-            .rsvd1 = 0,
+            }
-            .rsvd2 = 0,
+
-        }) catch return VkError.OutOfHostMemory;
+            var relocation_count: u32 = 0;
+            var relocs_ptr: u64 = 0;
+            for (groups.items) |group| {
+                if (group.source_handle == handle) {
+                    relocation_count = @intCast(group.entries.items.len);
+                    relocs_ptr = @intFromPtr(group.entries.items.ptr);
+                    break;
+                }
+            }
+            objects.append(allocator, .{
+                .handle = handle,
+                .relocation_count = relocation_count,
+                .relocs_ptr = relocs_ptr,
+                .alignment = 0,
+                .offset = 0,
+                .flags = flags,
+                .rsvd1 = 0,
+                .rsvd2 = 0,
+            }) catch return VkError.OutOfHostMemory;
+        }
 
         var exec_fences = std.ArrayList(_i915.ExecFence).empty;
         defer exec_fences.deinit(allocator);
@@ -142,7 +186,10 @@ pub const Device = struct {
             .DR4 = 0,
             .num_cliprects = @intCast(exec_fences.items.len),
             .cliprects_ptr = if (exec_fences.items.len == 0) 0 else @intFromPtr(exec_fences.items.ptr),
-            .flags = _i915.exec_blt | (if (exec_fences.items.len == 0) 0 else _i915.exec_fence_array),
+            .flags = @as(u64, switch (engine) {
+                .blitter => _i915.exec_blt,
+                .render => _i915.exec_render,
+            }) | (if (exec_fences.items.len == 0) 0 else _i915.exec_fence_array),
             .rsvd1 = 0,
             .rsvd2 = 0,
         };

src/intel/kmd.zig

@@ -19,12 +19,25 @@ pub const blt_depth_8: u32 = 0 << 24;
 pub const rop_source_copy: u32 = 0xcc << 16;
 pub const max_blt_span: vk.DeviceSize = 32 * 1024 - 1;
 
+pub const Engine = enum {
+    blitter,
+    render,
+};
+
+pub const Domain = enum {
+    none,
+    render,
+    instruction,
+};
+
 pub const Relocation = struct {
+    source_handle: ?u32 = null,
     target_handle: u32,
     offset: u64,
     delta: u32,
     read: bool = false,
     write: bool = false,
+    domain: Domain = .none,
 };
 
 pub const SyncDependency = struct {
@@ -63,10 +76,10 @@ pub const Device = union(KmdType) {
         };
     }
 
-    pub fn submitBatch(self: *Device, io: std.Io, allocator: std.mem.Allocator, commands: []const u32, relocations: []const Relocation, syncs: []const SyncDependency) VkError!void {
+    pub fn submitBatch(self: *Device, io: std.Io, allocator: std.mem.Allocator, engine: Engine, commands: []const u32, relocations: []const Relocation, syncs: []const SyncDependency) VkError!void {
         return switch (self.*) {
-            .i915 => |*device| device.submitBatch(io, allocator, commands, relocations, syncs),
+            .i915 => |*device| device.submitBatch(io, allocator, engine, commands, relocations, syncs),
-            .xe => |*device| device.submitBatch(io, allocator, commands, relocations, syncs),
+            .xe => |*device| device.submitBatch(io, allocator, engine, commands, relocations, syncs),
             .invalid => VkError.DeviceLost,
         };
     }

src/intel/lib.zig

@@ -2,15 +2,12 @@ const std = @import("std");
 const vk = @import("vulkan");
 pub const base = @import("base");
 
+pub const kmd = @import("kmd.zig");
+pub const compiler = @import("compiler/compiler.zig");
+
 pub const c = @import("intel_c");
 pub const config = base.config;
 
-pub const FlintInstance = @import("FlintInstance.zig");
-pub const FlintDevice = @import("FlintDevice.zig");
-pub const FlintPhysicalDevice = @import("FlintPhysicalDevice.zig");
-pub const FlintQueue = @import("FlintQueue.zig");
-pub const kmd = @import("kmd.zig");
-
 pub const FlintBinarySemaphore = @import("FlintBinarySemaphore.zig");
 pub const FlintBuffer = @import("FlintBuffer.zig");
 pub const FlintBufferView = @import("FlintBufferView.zig");
@@ -19,16 +16,20 @@ pub const FlintCommandPool = @import("FlintCommandPool.zig");
 pub const FlintDescriptorPool = @import("FlintDescriptorPool.zig");
 pub const FlintDescriptorSet = @import("FlintDescriptorSet.zig");
 pub const FlintDescriptorSetLayout = @import("FlintDescriptorSetLayout.zig");
+pub const FlintDevice = @import("FlintDevice.zig");
 pub const FlintDeviceMemory = @import("FlintDeviceMemory.zig");
 pub const FlintEvent = @import("FlintEvent.zig");
 pub const FlintFence = @import("FlintFence.zig");
 pub const FlintFramebuffer = @import("FlintFramebuffer.zig");
 pub const FlintImage = @import("FlintImage.zig");
 pub const FlintImageView = @import("FlintImageView.zig");
+pub const FlintInstance = @import("FlintInstance.zig");
+pub const FlintPhysicalDevice = @import("FlintPhysicalDevice.zig");
 pub const FlintPipeline = @import("FlintPipeline.zig");
 pub const FlintPipelineCache = @import("FlintPipelineCache.zig");
 pub const FlintPipelineLayout = @import("FlintPipelineLayout.zig");
 pub const FlintQueryPool = @import("FlintQueryPool.zig");
+pub const FlintQueue = @import("FlintQueue.zig");
 pub const FlintRenderPass = @import("FlintRenderPass.zig");
 pub const FlintSampler = @import("FlintSampler.zig");
 pub const FlintShaderModule = @import("FlintShaderModule.zig");
@@ -89,6 +90,7 @@ test {
     std.testing.refAllDecls(FlintRenderPass);
     std.testing.refAllDecls(FlintSampler);
     std.testing.refAllDecls(FlintShaderModule);
+    std.testing.refAllDecls(compiler);
     std.testing.refAllDecls(kmd);
     std.testing.refAllDecls(base);
 }

src/intel/pci_ids.zig

@@ -6,6 +6,69 @@ const PciInfo = struct {
 
 /// Not a hashmap as they need runtime allocations
 pub const map = [_]PciInfo{
+    .{ .id = 0x0a02, .name = "Intel(R) HD Graphics (Haswell-ULT GT1)", .is_discrete = false },
+    .{ .id = 0x0a06, .name = "Intel(R) HD Graphics (Haswell-ULT GT1)", .is_discrete = false },
+    .{ .id = 0x0a0a, .name = "Intel(R) HD Graphics (Haswell-ULT GT1)", .is_discrete = false },
+    .{ .id = 0x0a0b, .name = "Intel(R) HD Graphics (Haswell-ULT GT1)", .is_discrete = false },
+    .{ .id = 0x0a0e, .name = "Intel(R) HD Graphics (Haswell-ULX GT1)", .is_discrete = false },
+    .{ .id = 0x0402, .name = "Intel(R) HD Graphics (Haswell GT1)", .is_discrete = false },
+    .{ .id = 0x0406, .name = "Intel(R) HD Graphics (Haswell GT1)", .is_discrete = false },
+    .{ .id = 0x040a, .name = "Intel(R) HD Graphics (Haswell GT1)", .is_discrete = false },
+    .{ .id = 0x040b, .name = "Intel(R) HD Graphics (Haswell GT1)", .is_discrete = false },
+    .{ .id = 0x040e, .name = "Intel(R) HD Graphics (Haswell GT1)", .is_discrete = false },
+    .{ .id = 0x0c02, .name = "Intel(R) HD Graphics (Haswell-SDV GT1)", .is_discrete = false },
+    .{ .id = 0x0c06, .name = "Intel(R) HD Graphics (Haswell-SDV GT1)", .is_discrete = false },
+    .{ .id = 0x0c0a, .name = "Intel(R) HD Graphics (Haswell-SDV GT1)", .is_discrete = false },
+    .{ .id = 0x0c0b, .name = "Intel(R) HD Graphics (Haswell-SDV GT1)", .is_discrete = false },
+    .{ .id = 0x0c0e, .name = "Intel(R) HD Graphics (Haswell-SDV GT1)", .is_discrete = false },
+    .{ .id = 0x0d02, .name = "Intel(R) HD Graphics (Haswell-CRW GT1)", .is_discrete = false },
+    .{ .id = 0x0d06, .name = "Intel(R) HD Graphics (Haswell-CRW GT1)", .is_discrete = false },
+    .{ .id = 0x0d0a, .name = "Intel(R) HD Graphics (Haswell-CRW GT1)", .is_discrete = false },
+    .{ .id = 0x0d0b, .name = "Intel(R) HD Graphics (Haswell-CRW GT1)", .is_discrete = false },
+    .{ .id = 0x0d0e, .name = "Intel(R) HD Graphics (Haswell-CRW GT1)", .is_discrete = false },
+
+    .{ .id = 0x0a12, .name = "Intel(R) HD Graphics (Haswell-ULT GT2)", .is_discrete = false },
+    .{ .id = 0x0a16, .name = "Intel(R) HD Graphics 4400 (Haswell-ULT GT2)", .is_discrete = false },
+    .{ .id = 0x0a1a, .name = "Intel(R) HD Graphics (Haswell-ULT GT2)", .is_discrete = false },
+    .{ .id = 0x0a1b, .name = "Intel(R) HD Graphics (Haswell-ULT GT2)", .is_discrete = false },
+    .{ .id = 0x0a1e, .name = "Intel(R) HD Graphics 4200 (Haswell-ULX GT2)", .is_discrete = false },
+    .{ .id = 0x0412, .name = "Intel(R) HD Graphics 4600 (Haswell GT2)", .is_discrete = false },
+    .{ .id = 0x0416, .name = "Intel(R) HD Graphics 4600 (Haswell GT2)", .is_discrete = false },
+    .{ .id = 0x041a, .name = "Intel(R) HD Graphics P4600/P4700 (Haswell GT2)", .is_discrete = false },
+    .{ .id = 0x041b, .name = "Intel(R) HD Graphics (Haswell GT2)", .is_discrete = false },
+    .{ .id = 0x041e, .name = "Intel(R) HD Graphics 4400 (Haswell GT2)", .is_discrete = false },
+    .{ .id = 0x0c12, .name = "Intel(R) HD Graphics (Haswell-SDV GT2)", .is_discrete = false },
+    .{ .id = 0x0c16, .name = "Intel(R) HD Graphics (Haswell-SDV GT2)", .is_discrete = false },
+    .{ .id = 0x0c1a, .name = "Intel(R) HD Graphics (Haswell-SDV GT2)", .is_discrete = false },
+    .{ .id = 0x0c1b, .name = "Intel(R) HD Graphics (Haswell-SDV GT2)", .is_discrete = false },
+    .{ .id = 0x0c1e, .name = "Intel(R) HD Graphics (Haswell-SDV GT2)", .is_discrete = false },
+    .{ .id = 0x0d12, .name = "Intel(R) HD Graphics 4600 (Haswell-CRW GT2)", .is_discrete = false },
+    .{ .id = 0x0d16, .name = "Intel(R) HD Graphics (Haswell-CRW GT2)", .is_discrete = false },
+    .{ .id = 0x0d1a, .name = "Intel(R) HD Graphics (Haswell-CRW GT2)", .is_discrete = false },
+    .{ .id = 0x0d1b, .name = "Intel(R) HD Graphics (Haswell-CRW GT2)", .is_discrete = false },
+    .{ .id = 0x0d1e, .name = "Intel(R) HD Graphics (Haswell-CRW GT2)", .is_discrete = false },
+
+    .{ .id = 0x0a22, .name = "Intel(R) HD Graphics (Haswell-ULT GT3)", .is_discrete = false },
+    .{ .id = 0x0a26, .name = "Intel(R) HD Graphics 5000 (Haswell-ULT GT3)", .is_discrete = false },
+    .{ .id = 0x0a2a, .name = "Intel(R) HD Graphics (Haswell-ULT GT3)", .is_discrete = false },
+    .{ .id = 0x0a2b, .name = "Intel(R) HD Graphics (Haswell-ULT GT3)", .is_discrete = false },
+    .{ .id = 0x0a2e, .name = "Intel(R) Iris(R) Graphics 5100 (Haswell-ULT GT3)", .is_discrete = false },
+    .{ .id = 0x0422, .name = "Intel(R) HD Graphics (Haswell GT3)", .is_discrete = false },
+    .{ .id = 0x0426, .name = "Intel(R) HD Graphics (Haswell GT3)", .is_discrete = false },
+    .{ .id = 0x042a, .name = "Intel(R) HD Graphics (Haswell GT3)", .is_discrete = false },
+    .{ .id = 0x042b, .name = "Intel(R) HD Graphics (Haswell GT3)", .is_discrete = false },
+    .{ .id = 0x042e, .name = "Intel(R) HD Graphics (Haswell GT3)", .is_discrete = false },
+    .{ .id = 0x0c22, .name = "Intel(R) HD Graphics (Haswell-SDV GT3)", .is_discrete = false },
+    .{ .id = 0x0c26, .name = "Intel(R) HD Graphics (Haswell-SDV GT3)", .is_discrete = false },
+    .{ .id = 0x0c2a, .name = "Intel(R) HD Graphics (Haswell-SDV GT3)", .is_discrete = false },
+    .{ .id = 0x0c2b, .name = "Intel(R) HD Graphics (Haswell-SDV GT3)", .is_discrete = false },
+    .{ .id = 0x0c2e, .name = "Intel(R) HD Graphics (Haswell-SDV GT3)", .is_discrete = false },
+    .{ .id = 0x0d22, .name = "Intel(R) Iris(R) Pro Graphics 5200 (Haswell-CRW GT3)", .is_discrete = false },
+    .{ .id = 0x0d26, .name = "Intel(R) Iris(R) Pro Graphics P5200 (Haswell-CRW GT3)", .is_discrete = false },
+    .{ .id = 0x0d2a, .name = "Intel(R) HD Graphics (Haswell-CRW GT3)", .is_discrete = false },
+    .{ .id = 0x0d2b, .name = "Intel(R) HD Graphics (Haswell-CRW GT3)", .is_discrete = false },
+    .{ .id = 0x0d2e, .name = "Intel(R) HD Graphics (Haswell-CRW GT3)", .is_discrete = false },
+
     .{ .id = 0x1602, .name = "Intel(R) HD Graphics (Broadwell-U)", .is_discrete = false },
     .{ .id = 0x1606, .name = "Intel(R) HD Graphics (Broadwell-U GT1)", .is_discrete = false },
     .{ .id = 0x160a, .name = "Intel(R) HD Graphics (Broadwell-U)", .is_discrete = false },

src/intel/xe/kmd.zig

@@ -23,7 +23,15 @@ pub const Device = struct {
         return VkError.OutOfDeviceMemory;
     }
 
-    pub fn submitBatch(_: *Device, _: std.Io, _: std.mem.Allocator, _: []const u32, _: []const common_kmd.Relocation, _: []const common_kmd.SyncDependency) VkError!void {
+    pub fn submitBatch(
+        _: *Device,
+        _: std.Io,
+        _: std.mem.Allocator,
+        _: common_kmd.Engine,
+        _: []const u32,
+        _: []const common_kmd.Relocation,
+        _: []const common_kmd.SyncDependency,
+    ) VkError!void {
         return VkError.FeatureNotPresent;
     }
 };

src/phi/PhiBinarySemaphore.zig

@@ -8,6 +8,10 @@ const Self = @This();
 pub const Interface = base.BinarySemaphore;
 
 interface: Interface,
+mutex: std.Io.Mutex,
+condition: std.Io.Condition,
+is_signaled: bool,
+is_failed: bool,
 
 pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const vk.SemaphoreCreateInfo) VkError!*Self {
     const self = allocator.create(Self) catch return VkError.OutOfHostMemory;
@@ -23,6 +27,10 @@ pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const v
 
     self.* = .{
         .interface = interface,
+        .mutex = .init,
+        .condition = .init,
+        .is_signaled = false,
+        .is_failed = false,
     };
     return self;
 }
@@ -34,10 +42,39 @@ pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
 
 pub fn signal(interface: *Interface) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = self;
+    const io = interface.owner.io();
+
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+    defer self.mutex.unlock(io);
+
+    if (self.is_failed) return VkError.DeviceLost;
+    self.is_signaled = true;
+    self.condition.broadcast(io);
+}
+
+/// Latch an asynchronous queue/device failure and wake all host waiters
+pub fn fail(interface: *Interface) void {
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const io = interface.owner.io();
+
+    self.mutex.lock(io) catch return;
+    defer self.mutex.unlock(io);
+
+    self.is_failed = true;
+    self.condition.broadcast(io);
 }
 
 pub fn wait(interface: *Interface) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = self;
+    const io = interface.owner.io();
+
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+    defer self.mutex.unlock(io);
+
+    while (!self.is_signaled and !self.is_failed) {
+        self.condition.wait(io, &self.mutex) catch return VkError.DeviceLost;
+    }
+    if (self.is_failed) return VkError.DeviceLost;
+
+    self.is_signaled = false;
 }

src/phi/PhiBuffer.zig

@@ -1,7 +1,7 @@
 const std = @import("std");
 const vk = @import("vulkan");
 const base = @import("base");
-
+const proto = @import("lib.zig").proto;
 
 const VkError = base.VkError;
 
@@ -32,7 +32,6 @@ pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
     allocator.destroy(self);
 }
 
-pub fn getMemoryRequirements(interface: *Interface, requirements: *vk.MemoryRequirements) void {
+pub fn getMemoryRequirements(_: *Interface, requirements: *vk.MemoryRequirements) void {
-    _ = interface;
+    requirements.alignment = proto.PHI_MEMORY_ALIGNMENT;
-    _ = requirements;
 }

src/phi/PhiCommandBuffer.zig

@@ -6,6 +6,7 @@ const proto = lib.proto;
 
 const VkError = base.VkError;
 const PhiDeviceMemory = @import("PhiDeviceMemory.zig");
+const copy = @import("copy_commands.zig");
 
 const Self = @This();
 pub const Interface = base.CommandBuffer;
@@ -105,7 +106,7 @@ pub fn reset(interface: *Interface, flags: vk.CommandBufferResetFlags) VkError!v
     _ = flags;
 }
 
-fn appendCommand(self: *Self, comptime T: type, command_type: c_int, payload: T) VkError!void {
+pub fn appendCommand(self: *Self, comptime T: type, command_type: c_int, payload: T) VkError!void {
     const allocator = self.interface.host_allocator.allocator();
     const header: proto.PhiCmdHeader = .{
         .magic = proto.PHI_COMMAND_MAGIC,
@@ -117,15 +118,6 @@ fn appendCommand(self: *Self, comptime T: type, command_type: c_int, payload: T)
     self.serialized_cmd_count += 1;
 }
 
-fn remoteMemory(buffer: *base.Buffer) VkError!*PhiDeviceMemory {
-    const memory = buffer.memory orelse return VkError.ValidationFailed;
-    const phi_memory: *PhiDeviceMemory = @alignCast(@fieldParentPtr("interface", memory));
-    if (phi_memory.remote_handle == 0) {
-        return VkError.ValidationFailed;
-    }
-    return phi_memory;
-}
-
 pub fn beginQuery(interface: *Interface, pool: *base.QueryPool, query: u32, flags: vk.QueryControlFlags) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
     self.cmd_count += 1;
@@ -189,12 +181,12 @@ pub fn bindVertexBuffer(interface: *Interface, index: usize, buffer: *base.Buffe
 pub fn blitImage(interface: *Interface, src: *base.Image, src_layout: vk.ImageLayout, dst: *base.Image, dst_layout: vk.ImageLayout, regions: []const vk.ImageBlit, filter: vk.Filter) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
     self.cmd_count += 1;
-    _ = src;
+
     _ = src_layout;
-    _ = dst;
     _ = dst_layout;
-    _ = regions;
+
-    _ = filter;
+    for (regions) |region|
+        try copy.blitImage(self, src, dst, region, filter);
 }
 
 pub fn clearAttachment(interface: *Interface, attachment: vk.ClearAttachment, rect: vk.ClearRect) VkError!void {
@@ -224,52 +216,32 @@ pub fn clearDepthStencilImage(interface: *Interface, image: *base.Image, layout:
 
 pub fn copyBuffer(interface: *Interface, src: *base.Buffer, dst: *base.Buffer, regions: []const vk.BufferCopy) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    const src_memory = try remoteMemory(src);
+    try copy.copyBuffer(self, src, dst, regions);
-    const dst_memory = try remoteMemory(dst);
-
-    for (regions) |region| {
-        const src_offset, const src_overflow = @addWithOverflow(src.offset, region.src_offset);
-        const dst_offset, const dst_overflow = @addWithOverflow(dst.offset, region.dst_offset);
-        if (src_overflow != 0 or dst_overflow != 0) {
-            return VkError.ValidationFailed;
-        }
-
-        try self.appendCommand(proto.PhiCmdCopyBuffer, proto.PHI_CMD_COPY_BUFFER, .{
-            .size = region.size,
-            .src_memory = @intCast(src_memory.remote_handle),
-            .dst_memory = @intCast(dst_memory.remote_handle),
-            .src_offset = src_offset,
-            .dst_offset = dst_offset,
-        });
-    }
 }
 
 pub fn copyBufferToImage(interface: *Interface, src: *base.Buffer, dst: *base.Image, dst_layout: vk.ImageLayout, regions: []const vk.BufferImageCopy) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    self.cmd_count += 1;
-    _ = src;
-    _ = dst;
     _ = dst_layout;
-    _ = regions;
+
+    for (regions) |region|
+        try copy.copyBufferImage(self, src, dst, region, true);
 }
 
 pub fn copyImage(interface: *Interface, src: *base.Image, src_layout: vk.ImageLayout, dst: *base.Image, dst_layout: vk.ImageLayout, regions: []const vk.ImageCopy) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    self.cmd_count += 1;
-    _ = src;
     _ = src_layout;
-    _ = dst;
     _ = dst_layout;
-    _ = regions;
+
+    for (regions) |region|
+        try copy.copyImage(self, src, dst, region);
 }
 
 pub fn copyImageToBuffer(interface: *Interface, src: *base.Image, src_layout: vk.ImageLayout, dst: *base.Buffer, regions: []const vk.BufferImageCopy) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    self.cmd_count += 1;
-    _ = src;
     _ = src_layout;
-    _ = dst;
+
-    _ = regions;
+    for (regions) |region|
+        try copy.copyBufferImage(self, dst, src, region, false);
 }
 
 pub fn copyQueryPoolResults(interface: *Interface, pool: *base.QueryPool, first: u32, count: u32, dst: *base.Buffer, offset: vk.DeviceSize, stride: vk.DeviceSize, flags: vk.QueryResultFlags) VkError!void {
@@ -368,7 +340,8 @@ pub fn fillBuffer(interface: *Interface, buffer: *base.Buffer, offset: vk.Device
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
     self.cmd_count += 1;
 
-    const memory = try remoteMemory(buffer);
+    const memory_interface = buffer.memory orelse return VkError.ValidationFailed;
+    const memory: *PhiDeviceMemory = @alignCast(@fieldParentPtr("interface", memory_interface));
 
     try self.appendCommand(proto.PhiCmdFillBuffer, proto.PHI_CMD_FILL_BUFFER, .{
         .size = if (size == vk.WHOLE_SIZE) buffer.size - offset else size,

src/phi/PhiDescriptorPool.zig

@@ -4,7 +4,6 @@ const base = @import("base");
 
 const VkError = base.VkError;
 
-
 const Self = @This();
 pub const Interface = base.DescriptorPool;
 

src/phi/PhiDevice.zig

@@ -81,7 +81,7 @@ pub fn create(instance: *base.Instance, physical_device: *base.PhysicalDevice, a
     const phi_physical_device: *PhiPhysicalDevice = @alignCast(@fieldParentPtr("interface", physical_device));
 
     const transport = PhiTransport.init(instance, phi_physical_device.scif_node_id) catch blk: {
-        // If the first connection failed, upload and launch the daemon on the card.
+        // If the first connection failed, launch the daemon on the selected device.
         try uploadAndLaunchDaemon(instance, allocator, phi_physical_device.mic_device_num);
 
         const max_connect_attempts = 3;

src/phi/PhiDeviceMemory.zig

@@ -13,14 +13,42 @@ const Self = @This();
 pub const Interface = base.DeviceMemory;
 
 interface: Interface,
+
 remote_handle: u64,
+scif_offset: ?u64,
+
+/// Size of the region registered with SCIF.
+/// This is allocation size rounded up to page size.
+registered_size: usize,
+
+/// Bytes exposed through vkMapMemory.
+/// For HOST_VISIBLE memory this is a slice of host_backing.
 data: ?[]u8,
 
+/// Full page-aligned/page-rounded allocation registered with SCIF.
+host_backing: ?[]u8,
+
 pub fn create(device: *PhiDevice, allocator: std.mem.Allocator, size: vk.DeviceSize, memory_type_index: u32) VkError!*Self {
+    if (memory_type_index >= device.interface.physical_device.mem_props.memory_type_count) {
+        return VkError.ValidationFailed;
+    }
+
+    const allocation_size =
+        std.math.cast(usize, size) orelse return VkError.OutOfDeviceMemory;
+
+    const memory_type =
+        device.interface.physical_device.mem_props.memory_types[memory_type_index];
+
+    const host_visible = memory_type.property_flags.host_visible_bit;
+
     const self = allocator.create(Self) catch return VkError.OutOfHostMemory;
     errdefer allocator.destroy(self);
 
-    var interface = try Interface.init(&device.interface, size, memory_type_index);
+    var interface = try Interface.init(
+        &device.interface,
+        size,
+        memory_type_index,
+    );
 
     interface.vtable = &.{
         .destroy = destroy,
@@ -30,68 +58,109 @@ pub fn create(device: *PhiDevice, allocator: std.mem.Allocator, size: vk.DeviceS
         .invalidateRange = invalidateRange,
     };
 
-    if (memory_type_index >= device.interface.physical_device.mem_props.memory_type_count) {
+    if (host_visible) {
-        return VkError.ValidationFailed;
+        const page_size = std.heap.pageSize();
-    }
+        const registered_size = std.mem.alignForward(usize, allocation_size, page_size);
 
-    const memory_type = device.interface.physical_device.mem_props.memory_types[memory_type_index];
+        // This needs to be page aligned
-    const host_visible = memory_type.property_flags.host_visible_bit;
+        const backing = device.interface.device_allocator.allocator().alignedAlloc(u8, .fromByteUnits(std.heap.page_size_max), registered_size) catch return VkError.OutOfHostMemory;
-    const device_local = memory_type.property_flags.device_local_bit;
+        errdefer device.interface.device_allocator.allocator().free(backing);
-    const allocation_size = std.math.cast(usize, size) orelse return VkError.OutOfDeviceMemory;
 
-    const remote_handle = if (device_local) blk: {
+        const offset = device.transport.registerHostMemory(backing) catch return VkError.OutOfHostMemory;
-        const alloc_request: proto.PhiAllocMemoryRequest = .{
+        errdefer device.transport.unregisterHostMemory(offset, backing.len) catch @panic("Caught an error while handling an error");
-            .size = size,
+
-            .memory_type_index = memory_type_index,
+        const request: proto.PhiMapHostMemoryRequest = .{
-            .flags = 0,
+            .scif_offset = offset,
+            .scif_size = backing.len,
+            .size = allocation_size,
         };
 
-        var reply = std.mem.zeroes(proto.PhiAllocMemoryReply);
+        var reply = std.mem.zeroes(proto.PhiNewMemoryReply);
-        try device.transport.request(proto.PHI_PACKET_ALLOC_MEMORY, std.mem.asBytes(&alloc_request), std.mem.asBytes(&reply));
+
+        try device.transport.request(
+            proto.PHI_PACKET_MAP_HOST_MEMORY,
+            std.mem.asBytes(&request),
+            std.mem.asBytes(&reply),
+        );
 
         if (reply.result.status != proto.PHI_STATUS_OK) {
             return PhiTransport.statusToErr(reply.result.status);
         }
 
-        std.log.scoped(.PhiDeviceMemory).info("Recieved remote handle 0x{X}", .{reply.remote_handle});
+        self.* = .{
+            .interface = interface,
+            .remote_handle = reply.remote_handle,
+            .scif_offset = offset,
+            .registered_size = registered_size,
+            .data = backing[0..allocation_size],
+            .host_backing = backing,
+        };
+    } else {
+        const request: proto.PhiAllocMemoryRequest = .{
+            .size = size,
+            .memory_type_index = memory_type_index,
+            .flags = 0,
+        };
 
-        break :blk reply.remote_handle;
+        var reply = std.mem.zeroes(proto.PhiNewMemoryReply);
-    } else 0;
-    errdefer if (remote_handle != 0) self.interface.destroy(allocator);
 
-    const data = if (host_visible)
+        try device.transport.request(
-        device.interface.device_allocator.allocator().alloc(u8, allocation_size) catch return VkError.OutOfDeviceMemory
+            proto.PHI_PACKET_ALLOC_MEMORY,
-    else
+            std.mem.asBytes(&request),
-        null;
+            std.mem.asBytes(&reply),
+        );
 
-    self.* = .{
+        if (reply.result.status != proto.PHI_STATUS_OK) {
-        .interface = interface,
+            return PhiTransport.statusToErr(reply.result.status);
-        .remote_handle = remote_handle,
+        }
-        .data = data,
+
-    };
+        self.* = .{
+            .interface = interface,
+            .remote_handle = reply.remote_handle,
+            .scif_offset = null,
+            .registered_size = 0,
+            .data = null,
+            .host_backing = null,
+        };
+    }
 
     return self;
 }
 
 pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
-    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const self: *Self =
-    const device: *PhiDevice = @alignCast(@fieldParentPtr("interface", interface.owner));
+        @alignCast(@fieldParentPtr("interface", interface));
-    if (self.data) |data| {
+
-        interface.owner.device_allocator.allocator().free(data);
+    const device: *PhiDevice =
-    }
+        @alignCast(@fieldParentPtr("interface", interface.owner));
+
     if (self.remote_handle != 0) {
-        const request_payload: proto.PhiFreeMemoryRequest = .{
+        const request_payload: proto.PhiDestroyMemoryRequest = .{
             .remote_handle = self.remote_handle,
         };
-        var reply: proto.PhiFreeMemoryReply = undefined;
+
-        device.transport.request(proto.PHI_PACKET_FREE_MEMORY, std.mem.asBytes(&request_payload), std.mem.asBytes(&reply)) catch |err| {
+        var reply = std.mem.zeroes(proto.PhiResultReply);
-            std.log.scoped(.PhiTransport).err("Remote free failed: {s}", .{@errorName(err)});
+
+        device.transport.request(proto.PHI_PACKET_DESTROY_MEMORY, std.mem.asBytes(&request_payload), std.mem.asBytes(&reply)) catch |err| {
+            std.log.scoped(.PhiDeviceMemory).err("Remote free/unmap failed for handle 0x{X}: {s}", .{ self.remote_handle, @errorName(err) });
             return;
         };
+
         if (reply.result.status != proto.PHI_STATUS_OK) {
-            std.log.scoped(.PhiTransport).err("Remote free returned status {d}", .{reply.result.status});
+            std.log.scoped(.PhiDeviceMemory).err("Remote free/unmap for handle 0x{X} returned status {d}", .{ self.remote_handle, reply.result.status });
         }
     }
+
+    if (self.scif_offset) |scif_offset| {
+        device.transport.unregisterHostMemory(scif_offset, self.interface.size) catch |err| {
+            std.log.scoped(.PhiDeviceMemory).err("SCIF unregister failed: {s}", .{@errorName(err)});
+        };
+    }
+
+    if (self.host_backing) |host_backing| {
+        interface.owner.device_allocator.allocator().free(host_backing);
+    }
+
     allocator.destroy(self);
 }
 
@@ -109,19 +178,24 @@ pub fn invalidateRange(interface: *Interface, offset: vk.DeviceSize, size: vk.De
 
 pub fn map(interface: *Interface, offset: vk.DeviceSize, size: vk.DeviceSize) VkError![]u8 {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+
     const data = self.data orelse return VkError.MemoryMapFailed;
     const map_offset = std.math.cast(usize, offset) orelse return VkError.MemoryMapFailed;
+
     if (map_offset >= data.len) {
         return VkError.MemoryMapFailed;
     }
+
     const map_size = if (size == vk.WHOLE_SIZE)
         data.len - map_offset
     else
         std.math.cast(usize, size) orelse return VkError.MemoryMapFailed;
+
     if (map_size > data.len - map_offset) {
         return VkError.MemoryMapFailed;
     }
-    return data[map_offset..(map_offset + map_size)];
+
+    return data[map_offset .. map_offset + map_size];
 }
 
 pub fn unmap(_: *Interface) void {}

src/phi/PhiFence.zig

@@ -9,6 +9,10 @@ const Self = @This();
 pub const Interface = base.Fence;
 
 interface: Interface,
+mutex: std.Io.Mutex,
+condition: std.Io.Condition,
+is_signaled: bool,
+is_failed: bool,
 
 pub fn create(device: *Device, allocator: std.mem.Allocator, info: *const vk.FenceCreateInfo) VkError!*Self {
     const self = allocator.create(Self) catch return VkError.OutOfHostMemory;
@@ -26,6 +30,10 @@ pub fn create(device: *Device, allocator: std.mem.Allocator, info: *const vk.Fen
 
     self.* = .{
         .interface = interface,
+        .mutex = .init,
+        .condition = .init,
+        .is_signaled = info.flags.signaled_bit,
+        .is_failed = false,
     };
     return self;
 }
@@ -37,21 +45,84 @@ pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
 
 pub fn getStatus(interface: *Interface) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = self;
+    const io = interface.owner.io();
+
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+    defer self.mutex.unlock(io);
+
+    if (self.is_failed) return VkError.DeviceLost;
+    if (!self.is_signaled) return VkError.NotReady;
 }
 
 pub fn reset(interface: *Interface) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = self;
+    const io = interface.owner.io();
+
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+    defer self.mutex.unlock(io);
+
+    if (self.is_failed) return VkError.DeviceLost;
+    self.is_signaled = false;
 }
 
 pub fn signal(interface: *Interface) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = self;
+    const io = interface.owner.io();
+
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+    defer self.mutex.unlock(io);
+
+    if (self.is_failed) return VkError.DeviceLost;
+    self.is_signaled = true;
+    self.condition.broadcast(io);
+}
+
+/// Latch an asynchronous queue/device failure and wake all host waiters
+pub fn fail(interface: *Interface) void {
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const io = interface.owner.io();
+
+    self.mutex.lock(io) catch return;
+    defer self.mutex.unlock(io);
+
+    self.is_failed = true;
+    self.condition.broadcast(io);
 }
 
 pub fn wait(interface: *Interface, timeout: u64) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = self;
+    const io = interface.owner.io();
-    _ = timeout;
+
+    if (timeout == std.math.maxInt(@TypeOf(timeout))) {
+        self.mutex.lock(io) catch return VkError.DeviceLost;
+        defer self.mutex.unlock(io);
+
+        while (!self.is_signaled and !self.is_failed) {
+            self.condition.wait(io, &self.mutex) catch return VkError.DeviceLost;
+        }
+        if (self.is_failed) return VkError.DeviceLost;
+        return;
+    }
+
+    const deadline = std.Io.Clock.Timestamp.fromNow(io, .{
+        .raw = .fromNanoseconds(@intCast(timeout)),
+        .clock = .awake,
+    });
+    while (true) {
+        {
+            self.mutex.lock(io) catch return VkError.DeviceLost;
+            defer self.mutex.unlock(io);
+
+            if (self.is_failed) return VkError.DeviceLost;
+            if (self.is_signaled) return;
+        }
+
+        const remaining = deadline.durationFromNow(io);
+        if (remaining.raw.nanoseconds <= 0) return VkError.Timeout;
+
+        (std.Io.Clock.Duration{
+            .raw = .fromNanoseconds(@min(remaining.raw.nanoseconds, std.time.ns_per_ms)),
+            .clock = .awake,
+        }).sleep(io) catch return VkError.DeviceLost;
+    }
 }

src/phi/PhiImage.zig

@@ -1,15 +1,15 @@
 const std = @import("std");
 const vk = @import("vulkan");
 const base = @import("base");
+const proto = @import("lib.zig").proto;
+
+const PhiDeviceMemory = @import("PhiDeviceMemory.zig");
 
 const VkError = base.VkError;
 
 const Self = @This();
 pub const Interface = base.Image;
 
-pub const F32x4 = @Vector(4, f32);
-pub const U32x4 = @Vector(4, u32);
-
 interface: Interface,
 
 pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const vk.ImageCreateInfo) VkError!*Self {
@@ -17,6 +17,7 @@ pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const v
     errdefer allocator.destroy(self);
 
     var interface = try Interface.init(device, allocator, info);
+
     interface.vtable = &.{
         .destroy = destroy,
         .getMemoryRequirements = getMemoryRequirements,
@@ -39,39 +40,147 @@ pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
 }
 
 pub fn getMemoryRequirements(_: *Interface, requirements: *vk.MemoryRequirements) VkError!void {
-    _ = requirements;
+    requirements.alignment = proto.PHI_MEMORY_ALIGNMENT;
+    requirements.size = std.mem.alignForward(vk.DeviceSize, requirements.size, proto.PHI_MEMORY_ALIGNMENT);
 }
 
-pub fn copyToMemory(interface: *const Interface, memory: []u8, subresource: vk.ImageSubresourceLayers) VkError!void {
+pub fn copyToMemory(interface: *const Interface, dst: []u8, subresource: vk.ImageSubresourceLayers) VkError!void {
-    _ = interface;
+    const self: *const Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = subresource;
+    const memory_interface = interface.memory orelse return VkError.InvalidDeviceMemoryDrv;
-    @memset(memory, 0);
+    const memory: *PhiDeviceMemory = @alignCast(@fieldParentPtr("interface", memory_interface));
+    const data = memory.data orelse return VkError.InvalidDeviceMemoryDrv;
+
+    try validateSingleAspect(interface.format, subresource.aspect_mask);
+    if (subresource.mip_level >= interface.mip_levels or
+        subresource.base_array_layer >= interface.array_layers or
+        subresource.layer_count == 0)
+        return VkError.ValidationFailed;
+
+    const layer_count = if (subresource.layer_count == vk.REMAINING_ARRAY_LAYERS)
+        interface.array_layers - subresource.base_array_layer
+    else
+        subresource.layer_count;
+    if (layer_count > interface.array_layers - subresource.base_array_layer)
+        return VkError.ValidationFailed;
+
+    const level_size = self.getMultiSampledLevelSize(subresource.aspect_mask, subresource.mip_level);
+    const required_size, const size_overflow = @mulWithOverflow(level_size, @as(usize, layer_count));
+    if (size_overflow != 0 or dst.len < required_size)
+        return VkError.ValidationFailed;
+
+    const first_offset = try self.getSubresourceOffset(
+        subresource.aspect_mask,
+        subresource.mip_level,
+        subresource.base_array_layer,
+    );
+    const absolute_offset, const offset_overflow = @addWithOverflow(interface.memory_offset, first_offset);
+    if (offset_overflow != 0)
+        return VkError.ValidationFailed;
+
+    var src_offset = std.math.cast(usize, absolute_offset) orelse return VkError.InvalidDeviceMemoryDrv;
+    var dst_offset: usize = 0;
+    const layer_pitch = self.getLayerSize(subresource.aspect_mask);
+
+    for (0..layer_count) |_| {
+        if (src_offset > data.len or level_size > data.len - src_offset)
+            return VkError.InvalidDeviceMemoryDrv;
+        @memcpy(dst[dst_offset..][0..level_size], data[src_offset..][0..level_size]);
+        dst_offset += level_size;
+        src_offset += layer_pitch;
+    }
+}
+
+pub fn getSubresourceOffset(self: *const Self, aspect_mask: vk.ImageAspectFlags, mip_level: u32, layer: u32) VkError!usize {
+    if (mip_level >= self.interface.mip_levels or layer >= self.interface.array_layers)
+        return VkError.ValidationFailed;
+
+    var offset = try self.getAspectOffset(aspect_mask);
+    offset += layer * self.getLayerSize(aspect_mask);
+    for (0..mip_level) |mip|
+        offset += self.getMultiSampledLevelSize(aspect_mask, @intCast(mip));
+    return offset;
+}
+
+fn getAspectOffset(self: *const Self, aspect_mask: vk.ImageAspectFlags) VkError!usize {
+    try validateSingleAspect(self.interface.format, aspect_mask);
+    return switch (self.interface.format) {
+        .d16_unorm_s8_uint,
+        .d24_unorm_s8_uint,
+        .d32_sfloat_s8_uint,
+        => if (aspect_mask.stencil_bit)
+            self.interface.getTotalSizeForAspect(.{ .depth_bit = true })
+        else
+            0,
+        else => 0,
+    };
 }
 
 pub fn getTotalSizeForAspect(interface: *const Interface, aspect_mask: vk.ImageAspectFlags) VkError!usize {
-    _ = aspect_mask;
+    const self: *const Self = @alignCast(@fieldParentPtr("interface", interface));
-    return interface.extent.width * interface.extent.height * interface.extent.depth * base.format.texelSize(interface.format);
+    const valid_aspects = base.format.toAspect(interface.format);
+    if (aspect_mask.toInt() == 0 or aspect_mask.subtract(valid_aspects).toInt() != 0)
+        return VkError.ValidationFailed;
+
+    var size: usize = 0;
+    if (aspect_mask.color_bit)
+        size += self.getLayerSize(.{ .color_bit = true });
+    if (aspect_mask.depth_bit)
+        size += self.getLayerSize(.{ .depth_bit = true });
+    if (aspect_mask.stencil_bit)
+        size += self.getLayerSize(.{ .stencil_bit = true });
+    return size * interface.array_layers;
 }
 
 pub fn getSubresourceLayout(interface: *const Interface, subresource: vk.ImageSubresource) VkError!vk.SubresourceLayout {
-    _ = subresource;
+    const self: *const Self = @alignCast(@fieldParentPtr("interface", interface));
+    try validateSingleAspect(interface.format, subresource.aspect_mask);
+
     return .{
-        .offset = 0,
+        .offset = try self.getSubresourceOffset(subresource.aspect_mask, subresource.mip_level, subresource.array_layer),
-        .size = try getTotalSizeForAspect(interface, base.format.toAspect(interface.format)),
+        .size = self.getMultiSampledLevelSize(subresource.aspect_mask, subresource.mip_level),
-        .row_pitch = getRowPitchMemSizeForMipLevel(interface, base.format.toAspect(interface.format), 0),
+        .row_pitch = getRowPitchMemSizeForMipLevel(interface, subresource.aspect_mask, subresource.mip_level),
-        .array_pitch = getSliceMemSizeForMipLevel(interface, base.format.toAspect(interface.format), 0),
+        .array_pitch = self.getLayerSize(subresource.aspect_mask),
-        .depth_pitch = getSliceMemSizeForMipLevel(interface, base.format.toAspect(interface.format), 0),
+        .depth_pitch = getSliceMemSizeForMipLevel(interface, subresource.aspect_mask, subresource.mip_level),
+    };
+}
+
+pub fn getLayerSize(self: *const Self, aspect_mask: vk.ImageAspectFlags) usize {
+    var size: usize = 0;
+    for (0..self.interface.mip_levels) |mip_level|
+        size += self.getMultiSampledLevelSize(aspect_mask, @intCast(mip_level));
+    return size;
+}
+
+pub inline fn getMultiSampledLevelSize(self: *const Self, aspect_mask: vk.ImageAspectFlags, mip_level: u32) usize {
+    return self.getMipLevelSize(aspect_mask, mip_level) * self.interface.samples.toInt();
+}
+
+pub inline fn getMipLevelSize(self: *const Self, aspect_mask: vk.ImageAspectFlags, mip_level: u32) usize {
+    return getSliceMemSizeForMipLevel(&self.interface, aspect_mask, mip_level) * self.getMipLevelExtent(mip_level).depth;
+}
+
+pub fn getMipLevelExtent(self: *const Self, mip_level: u32) vk.Extent3D {
+    return .{
+        .width = @max(1, self.interface.extent.width >> @intCast(mip_level)),
+        .height = @max(1, self.interface.extent.height >> @intCast(mip_level)),
+        .depth = @max(1, self.interface.extent.depth >> @intCast(mip_level)),
     };
 }
 
 pub fn getSliceMemSizeForMipLevel(interface: *const Interface, aspect_mask: vk.ImageAspectFlags, mip_level: u32) usize {
-    _ = aspect_mask;
+    const self: *const Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = mip_level;
+    const extent = self.getMipLevelExtent(mip_level);
-    return interface.extent.width * interface.extent.height * base.format.texelSize(interface.format);
+    return base.format.sliceMemSize(base.format.fromAspect(interface.format, aspect_mask), extent.width, extent.height);
 }
 
 pub fn getRowPitchMemSizeForMipLevel(interface: *const Interface, aspect_mask: vk.ImageAspectFlags, mip_level: u32) usize {
-    _ = aspect_mask;
+    const self: *const Self = @alignCast(@fieldParentPtr("interface", interface));
-    _ = mip_level;
+    const extent = self.getMipLevelExtent(mip_level);
-    return interface.extent.width * base.format.texelSize(interface.format);
+    return base.format.pitchMemSize(base.format.fromAspect(interface.format, aspect_mask), extent.width);
+}
+
+fn validateSingleAspect(format: vk.Format, aspect_mask: vk.ImageAspectFlags) VkError!void {
+    const valid_aspects = base.format.toAspect(format);
+    if (aspect_mask.toInt() == 0 or @popCount(aspect_mask.toInt()) != 1 or aspect_mask.subtract(valid_aspects).toInt() != 0)
+        return VkError.ValidationFailed;
 }

src/phi/PhiInstance.zig

@@ -32,6 +32,8 @@ pub const extensions = [_]vk.ExtensionProperties{
     castExtension(vk.extensions.khr_get_physical_device_properties_2),
     castExtension(vk.extensions.khr_surface),
     castExtension(vk.extensions.khr_wayland_surface),
+    castExtension(vk.extensions.khr_xlib_surface),
+    castExtension(vk.extensions.khr_xcb_surface),
 };
 
 pub fn create(allocator: std.mem.Allocator, infos: *const vk.InstanceCreateInfo) VkError!*Interface {

src/phi/PhiPhysicalDevice.zig

@@ -33,7 +33,7 @@ interface: Interface,
 scif_node_id: u16,
 mic_device_num: u32,
 
-pub fn create(allocator: std.mem.Allocator, instance: *base.Instance, mic_device: mic.Device, mic_device_num: u32) VkError!*Self {
+pub fn create(allocator: std.mem.Allocator, instance: *base.Instance, mic_device: ?mic.Device, mic_device_num: u32) VkError!*Self {
     const self = allocator.create(Self) catch return VkError.OutOfHostMemory;
     errdefer allocator.destroy(self);
 
@@ -61,7 +61,8 @@ pub fn create(allocator: std.mem.Allocator, instance: *base.Instance, mic_device
 
     @memset(interface.props.device_name[0..], 0);
 
-    if (mic_device.pciConfig()) |pci_value| {
+    const device = mic_device.?;
+    if (device.pciConfig()) |pci_value| {
         var pci = pci_value;
         defer pci.deinit();
 
@@ -224,24 +225,23 @@ pub fn create(allocator: std.mem.Allocator, instance: *base.Instance, mic_device
         };
     }
 
-    if (mic_device.memoryInfo()) |memory_value| {
+    interface.mem_props.memory_heap_count = 2;
+    if (device.memoryInfo()) |memory_value| {
         var memory = memory_value;
         defer memory.deinit();
 
-        interface.mem_props.memory_heap_count = 2;
-
         interface.mem_props.memory_heaps[0] = .{
             .size = memory.size() catch 0,
             .flags = .{ .device_local_bit = true },
         };
-        interface.mem_props.memory_heaps[1] = .{
-            .size = std.process.totalSystemMemory() catch 0,
-            .flags = .{},
-        };
     } else |err| {
         std.log.scoped(.MIC).err("Failed to fetch device memory infos: {s}", .{@errorName(err)});
         return VkError.InitializationFailed;
     }
+    interface.mem_props.memory_heaps[1] = .{
+        .size = std.process.totalSystemMemory() catch 0,
+        .flags = .{},
+    };
 
     interface.features = .{
         .shader_float_64 = .true,

src/phi/PhiQueue.zig

@@ -4,8 +4,10 @@ const base = @import("base");
 const lib = @import("lib.zig");
 const proto = lib.proto;
 
+const PhiBinarySemaphore = @import("PhiBinarySemaphore.zig");
 const PhiCommandBuffer = @import("PhiCommandBuffer.zig");
 const PhiDevice = @import("PhiDevice.zig");
+const PhiFence = @import("PhiFence.zig");
 const PhiTransport = @import("PhiTransport.zig");
 
 const VkError = base.VkError;
@@ -13,7 +15,65 @@ const VkError = base.VkError;
 const Self = @This();
 pub const Interface = base.Queue;
 
+const ring_capacity: usize = @intCast(proto.PHI_QUEUE_RING_CAPACITY);
+const ring_capacity_u64: u64 = @intCast(ring_capacity);
+const shutdown_sequence = std.math.maxInt(u64);
+const shutdown_timeout_ns = 5 * std.time.ns_per_s;
+const shutdown_poll_ns = 10 * std.time.ns_per_ms;
+
+const CompletionShutdown = enum {
+    acknowledged,
+    stopped_without_acknowledgement,
+    timed_out,
+    wait_failed,
+};
+
+const PreparedSubmit = struct {
+    wait_semaphores: std.ArrayList(*base.BinarySemaphore),
+    signal_semaphores: std.ArrayList(*base.BinarySemaphore),
+    command_backing: ?[]u8,
+    scif_offset: ?u64,
+    registered_size: usize,
+    command_size: usize,
+    command_count: u64,
+};
+
+const PendingCompletion = struct {
+    signal_semaphores: std.ArrayList(*base.BinarySemaphore),
+    fence: ?*base.Fence,
+    command_backing: ?[]u8,
+    scif_offset: ?u64,
+    registered_size: usize,
+};
+
+const TaskData = struct {
+    queue: *Self,
+    sequence: usize,
+    submits: std.ArrayList(PreparedSubmit),
+    fence: ?*base.Fence,
+};
+
 interface: Interface,
+transport: PhiTransport,
+ring_backing: []u8,
+ring_offset: u64,
+shared: *proto.PhiQueueShared,
+
+submit_group: std.Io.Group,
+completion_group: std.Io.Group,
+mutex: std.Io.Mutex,
+condition: std.Io.Condition,
+
+next_task_sequence: usize,
+executing_task_sequence: usize,
+next_remote_sequence: u64,
+completed_sequence: u64,
+pending: [ring_capacity]?PendingCompletion,
+
+error_state: ?VkError,
+shutting_down: bool,
+completion_stopped: bool,
+remote_stopped: bool,
 
 pub fn create(allocator: std.mem.Allocator, device: *base.Device, index: u32, family_index: u32, flags: vk.DeviceQueueCreateFlags) VkError!*Interface {
     const self = allocator.create(Self) catch return VkError.OutOfHostMemory;
@@ -26,12 +86,155 @@ pub fn create(allocator: std.mem.Allocator, device: *base.Device, index: u32, fa
         .waitIdle = waitIdle,
     };
 
-    self.* = .{ .interface = interface };
+    const phi_device: *PhiDevice = @alignCast(@fieldParentPtr("interface", device));
+    var transport = try phi_device.transport.connectPeer();
+    errdefer transport.close();
+
+    const device_allocator = device.device_allocator.allocator();
+    const page_size = std.heap.pageSize();
+    const registered_size = std.mem.alignForward(usize, @sizeOf(proto.PhiQueueShared), page_size);
+    const ring_backing = device_allocator.alignedAlloc(
+        u8,
+        .fromByteUnits(std.heap.page_size_max),
+        registered_size,
+    ) catch return VkError.OutOfHostMemory;
+    errdefer device_allocator.free(ring_backing);
+    @memset(ring_backing, 0);
+
+    const ring_offset = try transport.registerHostMemory(ring_backing);
+    errdefer transport.unregisterHostMemory(ring_offset, ring_backing.len) catch @panic("Caught an error while handling an error");
+
+    const setup_request: proto.PhiQueueSetupRequest = .{
+        .scif_offset = ring_offset,
+        .scif_size = ring_backing.len,
+        .ring_capacity = @intCast(ring_capacity),
+        .reserved = 0,
+    };
+    var setup_reply = std.mem.zeroes(proto.PhiResultReply);
+    try transport.request(
+        proto.PHI_PACKET_QUEUE_SETUP,
+        std.mem.asBytes(&setup_request),
+        std.mem.asBytes(&setup_reply),
+    );
+    if (setup_reply.result.status != proto.PHI_STATUS_OK) {
+        return PhiTransport.statusToErr(setup_reply.result.status);
+    }
+
+    const shared: *proto.PhiQueueShared = @ptrCast(@alignCast(ring_backing.ptr));
+
+    self.* = .{
+        .interface = interface,
+        .transport = transport,
+        .ring_backing = ring_backing,
+        .ring_offset = ring_offset,
+        .shared = shared,
+        .submit_group = .init,
+        .completion_group = .init,
+        .mutex = .init,
+        .condition = .init,
+        .next_task_sequence = 0,
+        .executing_task_sequence = 0,
+        .next_remote_sequence = 1,
+        .completed_sequence = 0,
+        .pending = [_]?PendingCompletion{null} ** ring_capacity,
+        .error_state = null,
+        .shutting_down = false,
+        .completion_stopped = false,
+        .remote_stopped = false,
+    };
+
+    self.completion_group.async(device.io(), completionRunner, .{self});
     return &self.interface;
 }
 
 pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) VkError!void {
     const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const io = interface.owner.io();
+    const device_allocator = interface.owner.device_allocator.allocator();
+
+    var graceful_shutdown = true;
+    waitIdle(interface) catch |err| {
+        graceful_shutdown = false;
+        std.log.scoped(.PhiQueue).warn("Queue did not become idle during destruction: {s}", .{@errorName(err)});
+    };
+    var mutex_locked = true;
+    self.mutex.lock(io) catch {
+        mutex_locked = false;
+        graceful_shutdown = false;
+    };
+    var shutdown_next_sequence: u64 = 0;
+    var shutdown_completed_sequence: u64 = 0;
+    if (mutex_locked) {
+        self.shutting_down = true;
+        shutdown_next_sequence = self.next_remote_sequence;
+        shutdown_completed_sequence = self.completed_sequence;
+        self.condition.broadcast(io);
+        self.mutex.unlock(io);
+    }
+
+    if (graceful_shutdown) {
+        std.log.scoped(.PhiQueue).info(
+            "Sending shutdown doorbell (next remote sequence {d}, completed {d})",
+            .{ shutdown_next_sequence, shutdown_completed_sequence },
+        );
+        self.transport.sendQueueDoorbell(shutdown_sequence) catch |err| {
+            graceful_shutdown = false;
+            std.log.scoped(.PhiQueue).warn("Failed to send queue shutdown doorbell: {s}", .{@errorName(err)});
+        };
+
+        if (graceful_shutdown) {
+            switch (self.waitForCompletionShutdown(io)) {
+                .acknowledged => {},
+                .stopped_without_acknowledgement => {
+                    graceful_shutdown = false;
+                    std.log.scoped(.PhiQueue).warn("Remote queue stopped without acknowledging shutdown", .{});
+                },
+                .timed_out => {
+                    graceful_shutdown = false;
+                    const progress = self.completionProgress(io);
+                    std.log.scoped(.PhiQueue).warn(
+                        "Timed out waiting for remote queue shutdown (next remote sequence {d}, completed {d}); closing SCIF endpoint",
+                        .{ progress.next, progress.completed },
+                    );
+                },
+                .wait_failed => {
+                    graceful_shutdown = false;
+                    std.log.scoped(.PhiQueue).warn("Failed while waiting for remote queue shutdown; closing SCIF endpoint", .{});
+                },
+            }
+        }
+    }
+
+    if (!graceful_shutdown) {
+        // Wake the blocking completion receiver before releasing queue storage.
+        // Keep libscif loaded until the receiver has returned from scif_recv.
+        self.transport.interrupt();
+    }
+
+    self.completion_group.await(io) catch |err| {
+        graceful_shutdown = false;
+        std.log.scoped(.PhiQueue).warn("Failed while joining completion receiver: {s}", .{@errorName(err)});
+    };
+
+    if (graceful_shutdown) {
+        self.mutex.lock(io) catch {
+            graceful_shutdown = false;
+        };
+        if (graceful_shutdown) {
+            graceful_shutdown = self.remote_stopped;
+            self.mutex.unlock(io);
+        }
+    }
+
+    if (graceful_shutdown) {
+        self.transport.unregisterHostMemory(self.ring_offset, self.ring_backing.len) catch |err| {
+            std.log.scoped(.PhiQueue).warn("Failed to unregister queue ring: {s}", .{@errorName(err)});
+        };
+    }
+
+    self.transport.close();
+    cleanupPendingAfterClose(self, device_allocator);
+    device_allocator.free(self.ring_backing);
     allocator.destroy(self);
 }
 
@@ -43,46 +246,522 @@ pub fn bindSparse(interface: *Interface, info: []const vk.BindSparseInfo, fence:
 }
 
 pub fn submit(interface: *Interface, infos: []Interface.SubmitInfo, fence: ?*base.Fence) VkError!void {
-    const device: *PhiDevice = @alignCast(@fieldParentPtr("interface", interface.owner));
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const io = interface.owner.io();
+    const allocator = interface.owner.device_allocator.allocator();
 
-    for (infos) |info| {
+    try self.checkHealthy();
-        for (info.wait_semaphores.items) |semaphore| {
-            try semaphore.wait();
-        }
 
-        for (info.command_buffers.items) |command_buffer| {
+    const data = allocator.create(TaskData) catch return VkError.OutOfDeviceMemory;
-            const phi_command_buffer: *PhiCommandBuffer = @alignCast(@fieldParentPtr("interface", command_buffer));
+    errdefer allocator.destroy(data);
 
-            const work_execution_request: proto.PhiWorkExecutionRequest = .{
+    var prepared_submits = try prepareSubmits(self, allocator, infos);
-                .cmd_count = phi_command_buffer.serialized_cmd_count,
+    errdefer deinitPreparedSubmits(self, allocator, &prepared_submits);
-                .command_buffer_size = phi_command_buffer.commands.items.len,
-            };
-            const payload_size = @sizeOf(proto.PhiWorkExecutionRequest) + phi_command_buffer.commands.items.len;
-            const allocator = interface.host_allocator.allocator();
-            const payload = allocator.alloc(u8, payload_size) catch return VkError.OutOfHostMemory;
-            defer allocator.free(payload);
 
-            @memcpy(payload[0..@sizeOf(proto.PhiWorkExecutionRequest)], std.mem.asBytes(&work_execution_request));
+    const sequence = blk: {
-            @memcpy(payload[@sizeOf(proto.PhiWorkExecutionRequest)..], phi_command_buffer.commands.items);
+        self.mutex.lock(io) catch return VkError.DeviceLost;
+        defer self.mutex.unlock(io);
 
-            // Synchronous queues for now
+        if (self.error_state) |err| return err;
-            var reply = std.mem.zeroes(proto.PhiWorkExecutionReply);
+        if (self.shutting_down) return VkError.DeviceLost;
-            try device.transport.request(proto.PHI_PACKET_WORK_EXECUTION, payload, std.mem.asBytes(&reply));
 
-            if (reply.result.status != proto.PHI_STATUS_OK) {
+        const value = self.next_task_sequence;
-                return PhiTransport.statusToErr(reply.result.status);
+        self.next_task_sequence += 1;
-            }
+        break :blk value;
-        }
+    };
 
-        for (info.signal_semaphores.items) |semaphore| {
+    data.* = .{
-            try semaphore.signal();
+        .queue = self,
-        }
+        .sequence = sequence,
-    }
+        .submits = prepared_submits,
-    if (fence) |value| {
+        .fence = fence,
-        try value.signal();
+    };
-    }
+
+    self.submit_group.async(io, taskRunner, .{data});
 }
 
 pub fn waitIdle(interface: *Interface) VkError!void {
-    _ = interface;
+    const self: *Self = @alignCast(@fieldParentPtr("interface", interface));
+    const io = interface.owner.io();
+
+    self.submit_group.await(io) catch {
+        self.markLost(VkError.DeviceLost);
+        return VkError.DeviceLost;
+    };
+
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+    defer self.mutex.unlock(io);
+
+    while (self.completed_sequence + 1 < self.next_remote_sequence and self.error_state == null) {
+        self.condition.wait(io, &self.mutex) catch return VkError.DeviceLost;
+    }
+
+    if (self.error_state) |err| return err;
+}
+
+fn checkHealthy(self: *Self) VkError!void {
+    const io = self.interface.owner.io();
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+    defer self.mutex.unlock(io);
+
+    if (self.error_state) |err| return err;
+    if (self.shutting_down) return VkError.DeviceLost;
+}
+
+fn prepareSubmits(self: *Self, allocator: std.mem.Allocator, infos: []Interface.SubmitInfo) VkError!std.ArrayList(PreparedSubmit) {
+    var submits = std.ArrayList(PreparedSubmit).initCapacity(allocator, infos.len) catch return VkError.OutOfDeviceMemory;
+    errdefer deinitPreparedSubmits(self, allocator, &submits);
+
+    for (infos) |info| {
+        var prepared = try prepareSubmit(self, allocator, info);
+        submits.append(allocator, prepared) catch {
+            deinitPreparedSubmit(self, allocator, &prepared);
+            return VkError.OutOfDeviceMemory;
+        };
+    }
+
+    return submits;
+}
+
+fn prepareSubmit(self: *Self, allocator: std.mem.Allocator, info: Interface.SubmitInfo) VkError!PreparedSubmit {
+    var wait_semaphores = info.wait_semaphores.clone(allocator) catch return VkError.OutOfDeviceMemory;
+    errdefer wait_semaphores.deinit(allocator);
+
+    var signal_semaphores = info.signal_semaphores.clone(allocator) catch return VkError.OutOfDeviceMemory;
+    errdefer signal_semaphores.deinit(allocator);
+
+    var command_size: usize = 0;
+    var command_count: u64 = 0;
+    for (info.command_buffers.items) |command_buffer| {
+        const phi_command_buffer: *PhiCommandBuffer = @alignCast(@fieldParentPtr("interface", command_buffer));
+
+        if (phi_command_buffer.commands.items.len > std.math.maxInt(usize) - command_size) {
+            return VkError.OutOfHostMemory;
+        }
+        command_size += phi_command_buffer.commands.items.len;
+
+        const serialized_cmd_count: u64 = @intCast(phi_command_buffer.serialized_cmd_count);
+        if (serialized_cmd_count > std.math.maxInt(u64) - command_count) {
+            return VkError.OutOfHostMemory;
+        }
+        command_count += serialized_cmd_count;
+    }
+
+    var command_backing: ?[]u8 = null;
+    var scif_offset: ?u64 = null;
+    var registered_size: usize = 0;
+
+    if (command_size != 0) {
+        const page_size = std.heap.pageSize();
+        if (command_size > std.math.maxInt(usize) - (page_size - 1)) {
+            return VkError.OutOfHostMemory;
+        }
+        registered_size = std.mem.alignForward(usize, command_size, page_size);
+
+        const backing = allocator.alignedAlloc(
+            u8,
+            .fromByteUnits(std.heap.page_size_max),
+            registered_size,
+        ) catch return VkError.OutOfHostMemory;
+        errdefer allocator.free(backing);
+        @memset(backing, 0);
+
+        var write_offset: usize = 0;
+        for (info.command_buffers.items) |command_buffer| {
+            const phi_command_buffer: *PhiCommandBuffer = @alignCast(@fieldParentPtr("interface", command_buffer));
+            const commands = phi_command_buffer.commands.items;
+            @memcpy(backing[write_offset .. write_offset + commands.len], commands);
+            write_offset += commands.len;
+        }
+
+        const offset = try self.transport.registerHostMemory(backing);
+        errdefer self.transport.unregisterHostMemory(offset, backing.len) catch @panic("Caught an error while handling an error");
+
+        command_backing = backing;
+        scif_offset = offset;
+    }
+
+    return .{
+        .wait_semaphores = wait_semaphores,
+        .signal_semaphores = signal_semaphores,
+        .command_backing = command_backing,
+        .scif_offset = scif_offset,
+        .registered_size = registered_size,
+        .command_size = command_size,
+        .command_count = command_count,
+    };
+}
+
+fn deinitPreparedSubmits(self: *Self, allocator: std.mem.Allocator, submits: *std.ArrayList(PreparedSubmit)) void {
+    for (submits.items) |*prepared| {
+        deinitPreparedSubmit(self, allocator, prepared);
+    }
+    submits.deinit(allocator);
+}
+
+fn deinitPreparedSubmit(self: *Self, allocator: std.mem.Allocator, prepared: *PreparedSubmit) void {
+    prepared.wait_semaphores.deinit(allocator);
+    prepared.signal_semaphores.deinit(allocator);
+
+    if (prepared.scif_offset) |offset| {
+        self.transport.unregisterHostMemory(offset, prepared.registered_size) catch |err| {
+            std.log.scoped(.PhiQueue).warn("Failed to unregister staged command buffer: {s}", .{@errorName(err)});
+        };
+    }
+    if (prepared.command_backing) |backing| allocator.free(backing);
+
+    prepared.command_backing = null;
+    prepared.scif_offset = null;
+    prepared.registered_size = 0;
+    prepared.command_size = 0;
+    prepared.command_count = 0;
+}
+
+fn taskRunner(data: *TaskData) void {
+    const self = data.queue;
+    const io = self.interface.owner.io();
+    const allocator = self.interface.owner.device_allocator.allocator();
+
+    defer {
+        deinitPreparedSubmits(self, allocator, &data.submits);
+        allocator.destroy(data);
+    }
+
+    self.mutex.lock(io) catch {
+        failTask(data);
+        self.markLost(VkError.DeviceLost);
+        return;
+    };
+    while (data.sequence != self.executing_task_sequence and self.error_state == null) {
+        self.condition.wait(io, &self.mutex) catch {
+            self.mutex.unlock(io);
+            failTask(data);
+            self.markLost(VkError.DeviceLost);
+            return;
+        };
+    }
+    if (self.error_state != null) {
+        self.mutex.unlock(io);
+        failTask(data);
+        return;
+    }
+    self.mutex.unlock(io);
+
+    var task_error: ?VkError = null;
+
+    if (data.submits.items.len == 0) {
+        if (data.fence) |fence| {
+            var marker: PreparedSubmit = .{
+                .wait_semaphores = .empty,
+                .signal_semaphores = .empty,
+                .command_backing = null,
+                .scif_offset = null,
+                .registered_size = 0,
+                .command_size = 0,
+                .command_count = 0,
+            };
+            self.publish(&marker, fence) catch |err| {
+                task_error = err;
+            };
+            if (task_error == null) data.fence = null;
+        }
+    } else {
+        for (data.submits.items, 0..) |*prepared, info_index| {
+            for (prepared.wait_semaphores.items) |semaphore| {
+                semaphore.wait() catch |err| {
+                    task_error = err;
+                    break;
+                };
+            }
+            if (task_error != null) break;
+
+            const submission_fence = if (info_index + 1 == data.submits.items.len) data.fence else null;
+            self.publish(prepared, submission_fence) catch |err| {
+                task_error = err;
+                break;
+            };
+            if (submission_fence != null) data.fence = null;
+        }
+    }
+
+    if (task_error) |err| {
+        failTask(data);
+        self.markLost(err);
+        return;
+    }
+
+    self.mutex.lock(io) catch {
+        self.markLost(VkError.DeviceLost);
+        return;
+    };
+    self.executing_task_sequence += 1;
+    self.condition.broadcast(io);
+    self.mutex.unlock(io);
+}
+
+fn publish(self: *Self, prepared: *PreparedSubmit, fence: ?*base.Fence) VkError!void {
+    const io = self.interface.owner.io();
+
+    self.mutex.lock(io) catch return VkError.DeviceLost;
+
+    while ((self.next_remote_sequence - 1) - self.completed_sequence >= ring_capacity_u64 and self.error_state == null) {
+        self.condition.wait(io, &self.mutex) catch {
+            self.mutex.unlock(io);
+            return VkError.DeviceLost;
+        };
+    }
+
+    if (self.error_state) |err| {
+        self.mutex.unlock(io);
+        return err;
+    }
+    if (self.shutting_down) {
+        self.mutex.unlock(io);
+        return VkError.DeviceLost;
+    }
+
+    const sequence = self.next_remote_sequence;
+    if (sequence == shutdown_sequence) {
+        self.mutex.unlock(io);
+        return VkError.DeviceLost;
+    }
+    const slot: usize = @intCast((sequence - 1) % ring_capacity_u64);
+    if (self.pending[slot] != null) {
+        self.mutex.unlock(io);
+        return VkError.DeviceLost;
+    }
+
+    const command_scif_offset = prepared.scif_offset orelse 0;
+    self.pending[slot] = .{
+        .signal_semaphores = prepared.signal_semaphores,
+        .fence = fence,
+        .command_backing = prepared.command_backing,
+        .scif_offset = prepared.scif_offset,
+        .registered_size = prepared.registered_size,
+    };
+
+    prepared.signal_semaphores = .empty;
+    prepared.command_backing = null;
+    prepared.scif_offset = null;
+    prepared.registered_size = 0;
+
+    self.shared.submissions[slot] = .{
+        .sequence = sequence,
+        .command_scif_offset = command_scif_offset,
+        .command_size = prepared.command_size,
+        .command_count = prepared.command_count,
+    };
+
+    @atomicStore(
+        @TypeOf(self.shared.producer_sequence),
+        &self.shared.producer_sequence,
+        @intCast(sequence),
+        .release,
+    );
+    self.next_remote_sequence += 1;
+    self.mutex.unlock(io);
+
+    self.transport.sendQueueDoorbell(sequence) catch |err| {
+        self.markLost(VkError.DeviceLost);
+        return err;
+    };
+}
+
+fn completionRunner(self: *Self) void {
+    defer self.markCompletionStopped();
+
+    while (true) {
+        const completion = self.transport.receiveQueueCompletion() catch |err| {
+            if (!self.isShuttingDown()) {
+                std.log.scoped(.PhiQueue).err("Queue completion receive failed: {s}", .{@errorName(err)});
+                self.markLost(VkError.DeviceLost);
+            } else {
+                std.log.scoped(.PhiQueue).warn("Queue completion receiver stopped without shutdown acknowledgement: {s}", .{@errorName(err)});
+            }
+            return;
+        };
+
+        if (completion.sequence == shutdown_sequence) {
+            const io = self.interface.owner.io();
+            self.mutex.lock(io) catch return;
+            self.remote_stopped = completion.status == proto.PHI_STATUS_OK;
+            std.log.scoped(.PhiQueue).info("Received remote queue shutdown acknowledgement (status {d})", .{completion.status});
+            self.condition.broadcast(io);
+            self.mutex.unlock(io);
+            return;
+        }
+
+        self.completeOne(completion);
+    }
+}
+
+fn completeOne(self: *Self, completion: proto.PhiQueueCompletion) void {
+    const io = self.interface.owner.io();
+    const allocator = self.interface.owner.device_allocator.allocator();
+
+    self.mutex.lock(io) catch {
+        self.markLost(VkError.DeviceLost);
+        return;
+    };
+
+    if (completion.sequence != self.completed_sequence + 1 or completion.sequence >= self.next_remote_sequence) {
+        self.mutex.unlock(io);
+        self.markLost(VkError.DeviceLost);
+        return;
+    }
+
+    const slot: usize = @intCast((completion.sequence - 1) % ring_capacity_u64);
+    var pending = self.pending[slot] orelse {
+        self.mutex.unlock(io);
+        self.markLost(VkError.DeviceLost);
+        return;
+    };
+    self.pending[slot] = null;
+    self.mutex.unlock(io);
+
+    var cleanup_failed = false;
+    if (pending.scif_offset) |offset| {
+        self.transport.unregisterHostMemory(offset, pending.registered_size) catch |err| {
+            cleanup_failed = true;
+            std.log.scoped(.PhiQueue).err("Failed to unregister completed command buffer: {s}", .{@errorName(err)});
+        };
+    }
+    if (pending.command_backing) |backing| allocator.free(backing);
+
+    if (completion.status != proto.PHI_STATUS_OK or cleanup_failed) {
+        std.log.scoped(.PhiQueue).err(
+            "Queue completion {d} failed with remote status {d} (cleanup failed: {})",
+            .{ completion.sequence, completion.status, cleanup_failed },
+        );
+        // A remote command error is confined to this submission. The protocol
+        // stream and ring remain synchronized, so poisoning every later CTS
+        // submission would only hide the command that actually failed.
+        if (cleanup_failed) self.markLost(VkError.DeviceLost);
+        failPending(&pending);
+    } else if (self.hasError()) {
+        failPending(&pending);
+    } else {
+        signalPending(self, &pending);
+    }
+
+    pending.signal_semaphores.deinit(allocator);
+
+    self.mutex.lock(io) catch {
+        self.markLost(VkError.DeviceLost);
+        return;
+    };
+    self.completed_sequence = completion.sequence;
+    self.condition.broadcast(io);
+    self.mutex.unlock(io);
+}
+
+fn signalPending(self: *Self, pending: *PendingCompletion) void {
+    var signal_failed = false;
+
+    for (pending.signal_semaphores.items) |semaphore| {
+        semaphore.signal() catch {
+            signal_failed = true;
+        };
+    }
+    if (pending.fence) |fence| {
+        fence.signal() catch {
+            signal_failed = true;
+        };
+    }
+
+    if (signal_failed) self.markLost(VkError.DeviceLost);
+}
+
+fn failPending(pending: *PendingCompletion) void {
+    for (pending.signal_semaphores.items) |semaphore| {
+        PhiBinarySemaphore.fail(semaphore);
+    }
+    if (pending.fence) |fence| PhiFence.fail(fence);
+}
+
+fn failTask(data: *TaskData) void {
+    for (data.submits.items) |*prepared| {
+        for (prepared.signal_semaphores.items) |semaphore| {
+            PhiBinarySemaphore.fail(semaphore);
+        }
+    }
+    if (data.fence) |fence| PhiFence.fail(fence);
+}
+
+fn markLost(self: *Self, _: VkError) void {
+    const io = self.interface.owner.io();
+    self.mutex.lock(io) catch return;
+    defer self.mutex.unlock(io);
+
+    const first_failure = self.error_state == null;
+    self.error_state = VkError.DeviceLost;
+    self.condition.broadcast(io);
+
+    if (!first_failure) return;
+
+    for (&self.pending) |*entry| {
+        if (entry.*) |*pending| failPending(pending);
+    }
+}
+
+fn hasError(self: *Self) bool {
+    const io = self.interface.owner.io();
+    self.mutex.lock(io) catch return true;
+    defer self.mutex.unlock(io);
+    return self.error_state != null;
+}
+
+fn completionProgress(self: *Self, io: std.Io) struct { next: u64, completed: u64 } {
+    self.mutex.lock(io) catch return .{ .next = 0, .completed = 0 };
+    defer self.mutex.unlock(io);
+    return .{ .next = self.next_remote_sequence, .completed = self.completed_sequence };
+}
+
+fn waitForCompletionShutdown(self: *Self, io: std.Io) CompletionShutdown {
+    const deadline = std.Io.Clock.Timestamp.fromNow(io, .{
+        .raw = .fromNanoseconds(shutdown_timeout_ns),
+        .clock = .awake,
+    });
+
+    while (true) {
+        self.mutex.lock(io) catch return .wait_failed;
+        const stopped = self.completion_stopped;
+        const acknowledged = self.remote_stopped;
+        self.mutex.unlock(io);
+
+        if (stopped) return if (acknowledged) .acknowledged else .stopped_without_acknowledgement;
+
+        const remaining = deadline.durationFromNow(io);
+        if (remaining.raw.nanoseconds <= 0) return .timed_out;
+        (std.Io.Clock.Duration{
+            .raw = .fromNanoseconds(@min(remaining.raw.nanoseconds, shutdown_poll_ns)),
+            .clock = .awake,
+        }).sleep(io) catch return .wait_failed;
+    }
+}
+
+fn markCompletionStopped(self: *Self) void {
+    const io = self.interface.owner.io();
+    self.mutex.lock(io) catch return;
+    self.completion_stopped = true;
+    self.condition.broadcast(io);
+    self.mutex.unlock(io);
+}
+
+fn isShuttingDown(self: *Self) bool {
+    const io = self.interface.owner.io();
+    self.mutex.lock(io) catch return true;
+    defer self.mutex.unlock(io);
+    return self.shutting_down;
+}
+
+fn cleanupPendingAfterClose(self: *Self, allocator: std.mem.Allocator) void {
+    for (&self.pending) |*entry| {
+        if (entry.*) |*pending| {
+            failPending(pending);
+            if (pending.command_backing) |backing| allocator.free(backing);
+            pending.signal_semaphores.deinit(allocator);
+            entry.* = null;
+        }
+    }
 }

src/phi/PhiShaderModule.zig

@@ -15,6 +15,7 @@ pub fn create(device: *base.Device, allocator: std.mem.Allocator, info: *const v
     errdefer allocator.destroy(self);
 
     var interface = try Interface.init(device, allocator, info);
+    errdefer interface.deinit();
     interface.vtable = &.{ .destroy = destroy };
 
     self.* = .{
@@ -30,6 +31,7 @@ pub fn destroy(interface: *Interface, allocator: std.mem.Allocator) void {
 }
 
 pub fn drop(self: *Self, allocator: std.mem.Allocator) void {
+    self.interface.deinit();
     allocator.destroy(self);
 }
 
@@ -38,7 +40,6 @@ pub fn ref(self: *Self) void {
 }
 
 pub fn unref(self: *Self, allocator: std.mem.Allocator) void {
-    if (self.ref_count.fetchSub(1, .release) == 1) {
+    if (self.ref_count.fetchSub(1, .acq_rel) == 1)
         self.drop(allocator);
-    }
 }

src/phi/PhiTransport.zig

@@ -5,38 +5,50 @@ const scif = @import("scif.zig");
 
 const VkError = base.VkError;
 const proto = lib.proto;
+const Endpoint = scif.epd_t;
 
 const Self = @This();
 
-epd: scif.epd_t,
+epd: Endpoint,
 sequence: u64 = 1,
 mutex: std.Io.Mutex = .init,
+endpoint_mutex: base.SpinMutex = .{},
+library_loaded: bool = true,
 instance: *base.Instance,
+node_id: u16,
 
 pub fn init(instance: *base.Instance, node_id: u16) VkError!Self {
-    try scif.load();
+    const epd = blk: {
-    errdefer scif.unload();
+        try scif.load();
+        errdefer scif.unload();
 
-    const epd = scif.open();
+        const endpoint = scif.open();
-    if (epd < 0) {
+        if (endpoint < 0) {
-        std.log.scoped(.PhiTransport).err("SCIF open failed", .{});
+            std.log.scoped(.PhiTransport).err("SCIF open failed", .{});
-        return VkError.InitializationFailed;
+            return VkError.InitializationFailed;
-    }
+        }
-    errdefer _ = scif.close(epd);
+        errdefer _ = scif.close(endpoint);
 
-    var dst: scif.PortId = .{
+        var dst: scif.PortId = .{
-        .node = node_id,
+            .node = node_id,
-        .port = @intCast(proto.PHI_SCIF_PORT),
+            .port = @intCast(proto.PHI_SCIF_PORT),
+        };
+
+        if (scif.connect(endpoint, &dst) < 0) {
+            std.log.scoped(.PhiTransport).err("SCIF connection to node {d} port {d} failed", .{ dst.node, dst.port });
+            return VkError.InitializationFailed;
+        }
+        break :blk endpoint;
     };
-
+    errdefer {
-    if (scif.connect(epd, &dst) < 0) {
+        closeEndpoint(epd);
-        std.log.scoped(.PhiTransport).err("SCIF connection to node {d} port {d} failed", .{ dst.node, dst.port });
+        scif.unload();
-        return VkError.InitializationFailed;
     }
 
     var self: Self = .{
         .epd = epd,
         .instance = instance,
+        .node_id = node_id,
     };
     try self.handshake();
 
@@ -44,17 +56,42 @@ pub fn init(instance: *base.Instance, node_id: u16) VkError!Self {
     return self;
 }
 
+pub fn connectPeer(self: *const Self) VkError!Self {
+    return init(self.instance, self.node_id);
+}
+
 pub fn deinit(self: *Self) void {
     var reply: proto.PhiResult = undefined;
     self.request(proto.PHI_PACKET_SHUTDOWN, &.{}, std.mem.asBytes(&reply)) catch |err| {
         std.log.scoped(.PhiTransport).warn("Failed to shut down remote session: {s}", .{@errorName(err)});
     };
 
-    _ = scif.close(self.epd);
+    self.close();
-    scif.unload();
     std.log.scoped(.PhiTransport).info("Closed connection", .{});
 }
 
+/// Close the endpoint so a thread blocked in SCIF receive wakes up. The SCIF
+/// library stays loaded until `close`, because that thread may still be
+/// returning through a dynamically loaded function.
+pub fn interrupt(self: *Self) void {
+    self.endpoint_mutex.lock();
+    const endpoint = self.epd;
+    self.epd = -1;
+    self.endpoint_mutex.unlock();
+
+    if (endpoint >= 0) closeEndpoint(endpoint);
+}
+
+/// Close a transport without issuing an RPC shutdown. Queue transports switch
+/// to a raw full-duplex doorbell protocol after setup and must use this path.
+pub fn close(self: *Self) void {
+    self.interrupt();
+    if (!self.library_loaded) return;
+
+    self.library_loaded = false;
+    scif.unload();
+}
+
 pub fn request(self: *Self, command: c_uint, payload: []const u8, reply_payload: []u8) VkError!void {
     self.mutex.lock(self.instance.io()) catch return VkError.DeviceLost;
     defer self.mutex.unlock(self.instance.io());
@@ -90,36 +127,63 @@ pub fn request(self: *Self, command: c_uint, payload: []const u8, reply_payload:
     try self.readAll(reply_payload);
 }
 
+pub fn sendQueueDoorbell(self: *Self, sequence: u64) VkError!void {
+    const doorbell: proto.PhiQueueDoorbell = .{
+        .sequence = sequence,
+    };
+    try self.writeAll(std.mem.asBytes(&doorbell));
+}
+
+pub fn receiveQueueCompletion(self: *Self) VkError!proto.PhiQueueCompletion {
+    // SAFETY: readAll initializes the complete structure.
+    var completion: proto.PhiQueueCompletion = undefined;
+    try self.readAll(std.mem.asBytes(&completion));
+    return completion;
+}
+
 pub fn statusToErr(status: c_int) VkError {
     return switch (status) {
         proto.PHI_STATUS_OUT_OF_MEMORY => VkError.OutOfDeviceMemory,
         proto.PHI_STATUS_UNSUPPORTED_VERSION => VkError.InitializationFailed,
+        proto.PHI_STATUS_INVALID_ARGUMENT => VkError.ValidationFailed,
         else => VkError.Unknown,
     };
 }
 
 fn writeAll(self: *Self, bytes: []const u8) VkError!void {
+    const endpoint = self.getEndpoint() orelse return VkError.DeviceLost;
     var offset: usize = 0;
     while (offset < bytes.len) {
-        const written = scif.send(self.epd, bytes[offset..].ptr, bytes.len - offset, scif.send_block);
+        const written = scif.send(endpoint, bytes[offset..].ptr, bytes.len - offset, scif.send_block);
         if (written <= 0) {
-            return VkError.InitializationFailed;
+            return VkError.DeviceLost;
         }
         offset += @intCast(written);
     }
 }
 
 fn readAll(self: *Self, bytes: []u8) VkError!void {
+    const endpoint = self.getEndpoint() orelse return VkError.DeviceLost;
     var offset: usize = 0;
     while (offset < bytes.len) {
-        const read = scif.recv(self.epd, bytes[offset..].ptr, bytes.len - offset, scif.recv_block);
+        const read = scif.recv(endpoint, bytes[offset..].ptr, bytes.len - offset, scif.recv_block);
         if (read <= 0) {
-            return VkError.InitializationFailed;
+            return VkError.DeviceLost;
         }
         offset += @intCast(read);
     }
 }
 
+fn getEndpoint(self: *Self) ?Endpoint {
+    self.endpoint_mutex.lock();
+    defer self.endpoint_mutex.unlock();
+    return if (self.epd >= 0) self.epd else null;
+}
+
+fn closeEndpoint(endpoint: Endpoint) void {
+    _ = scif.close(endpoint);
+}
+
 fn handshake(self: *Self) VkError!void {
     const request_payload: proto.PhiHelloRequest = .{
         .host_protocol_version = proto.PHI_PROTOCOL_VERSION,
@@ -137,3 +201,26 @@ fn handshake(self: *Self) VkError!void {
         return VkError.InitializationFailed;
     }
 }
+
+pub fn registerHostMemory(self: *Self, memory: []u8) VkError!u64 {
+    const endpoint = self.getEndpoint() orelse return VkError.DeviceLost;
+    const offset = scif.register(
+        endpoint,
+        memory.ptr,
+        memory.len,
+        0,
+        @intFromEnum(scif.Prot.read) | @intFromEnum(scif.Prot.write),
+        0,
+    );
+    if (offset < 0) {
+        return VkError.Unknown;
+    }
+    return @intCast(offset);
+}
+
+pub fn unregisterHostMemory(self: *Self, offset: u64, size: usize) VkError!void {
+    const endpoint = self.getEndpoint() orelse return VkError.DeviceLost;
+    if (scif.unregister(endpoint, @intCast(offset), size) != 0) {
+        return VkError.Unknown;
+    }
+}

src/phi/compiler/abi.zig

@@ -0,0 +1,25 @@
+pub const version: u32 = 1;
+
+pub const BufferDescriptor = extern struct {
+    address: u64,
+    size: u64,
+};
+
+pub const KernelContext = extern struct {
+    abi_version: u32,
+    resource_count: u32,
+    resources: u64,
+    push_constants: u64,
+    push_constant_size: u32,
+    reserved: u32 = 0,
+    base_group: [3]u32,
+    group_count: [3]u32,
+    local_size: [3]u32,
+    num_workgroups: [3]u32,
+};
+
+pub const EntryPoint = *const fn (
+    context: *const KernelContext,
+    begin_workgroup: u64,
+    end_workgroup: u64,
+) callconv(.c) void;

src/phi/compiler/analysis.zig

@@ -0,0 +1,21 @@
+const std = @import("std");
+const block_layout = @import("block_layout.zig");
+const edge_copies = @import("edge_copies.zig");
+const liveness = @import("liveness.zig");
+const register_allocator = @import("register_allocator.zig");
+
+pub const Analysis = struct {
+    layout: block_layout.Layout = block_layout.Layout.empty(),
+    liveness: liveness.Analysis = liveness.Analysis.empty(),
+    allocation: register_allocator.Allocation = register_allocator.Allocation.empty(),
+    edge_copy_plans: []edge_copies.Plan = &.{},
+
+    pub fn deinit(self: *Analysis, allocator: std.mem.Allocator) void {
+        self.layout.deinit(allocator);
+        self.liveness.deinit(allocator);
+        self.allocation.deinit(allocator);
+        for (self.edge_copy_plans) |*plan| plan.deinit(allocator);
+        if (self.edge_copy_plans.len != 0) allocator.free(self.edge_copy_plans);
+        self.* = undefined;
+    }
+};

src/phi/compiler/artifact.zig

@@ -0,0 +1,26 @@
+const std = @import("std");
+const abi = @import("abi.zig");
+
+pub const ResourceBinding = struct {
+    set: u32,
+    binding: u32,
+};
+
+pub const KernelInfo = struct {
+    abi_version: u32 = abi.version,
+    workgroup_size: [3]u32,
+    dispatch_width: u8,
+    stack_size: u32,
+    resources: []const ResourceBinding,
+};
+
+pub const Artifact = struct {
+    code: []u8,
+    info: KernelInfo,
+
+    pub fn deinit(self: *Artifact, allocator: std.mem.Allocator) void {
+        allocator.free(self.code);
+        allocator.free(self.info.resources);
+        self.* = undefined;
+    }
+};

src/phi/compiler/block_layout.zig

@@ -0,0 +1,17 @@
+const std = @import("std");
+const shader_ir = @import("shader_ir").ir;
+
+pub const Layout = struct {
+    blocks: []shader_ir.id.BlockId,
+    positions: []?usize,
+
+    pub fn empty() Layout {
+        return .{ .blocks = &.{}, .positions = &.{} };
+    }
+
+    pub fn deinit(self: *Layout, allocator: std.mem.Allocator) void {
+        if (self.blocks.len != 0) allocator.free(self.blocks);
+        if (self.positions.len != 0) allocator.free(self.positions);
+        self.* = undefined;
+    }
+};