fixing Phi device upload, adding actions for bidirectionnal git mirrors
Build / build (push) Failing after 58s
Test / build_and_test (push) Failing after 1m6s

This commit is contained in:
2026-07-23 22:26:43 +02:00
parent 9b1b223186
commit 9a2cd20019
11 changed files with 1126 additions and 523 deletions
+190
View File
@@ -0,0 +1,190 @@
name: Mirror Gitea refs to GitHub
on:
push:
branches:
- main
tags:
- "*"
jobs:
mirror:
runs-on: ubuntu-latest
env:
MIRROR_REPOSITORY: ${{ vars.MIRROR_REPOSITORY }}
MIRROR_BASE_BRANCH: ${{ vars.MIRROR_BASE_BRANCH }}
MIRROR_GITHUB_SSH_KEY: ${{ secrets.MIRROR_GITHUB_SSH_KEY }}
MIRROR_GITHUB_KNOWN_HOSTS: ${{ secrets.MIRROR_GITHUB_KNOWN_HOSTS }}
MIRROR_REF_TYPE: ${{ gitea.ref_type }}
MIRROR_REF_NAME: ${{ gitea.ref_name }}
steps:
- name: Check out the authoritative Gitea repository
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Push without rewriting GitHub history
shell: bash
run: |
set -Eeuo pipefail
RequireVariable() {
local variable_name="$1"
if [[ -z "${!variable_name:-}" ]]; then
echo "Missing required variable: ${variable_name}" >&2
exit 1
fi
}
for variable_name in \
MIRROR_REPOSITORY \
MIRROR_BASE_BRANCH \
MIRROR_GITHUB_SSH_KEY \
MIRROR_GITHUB_KNOWN_HOSTS \
MIRROR_REF_TYPE \
MIRROR_REF_NAME; do
RequireVariable "${variable_name}"
done
install -d -m 700 "${HOME}/.ssh"
printf '%s\n' "${MIRROR_GITHUB_SSH_KEY}" \
> "${HOME}/.ssh/id_ed25519"
printf '%s\n' "${MIRROR_GITHUB_KNOWN_HOSTS}" \
> "${HOME}/.ssh/known_hosts"
chmod 600 "${HOME}/.ssh/id_ed25519"
chmod 644 "${HOME}/.ssh/known_hosts"
git remote remove github 2>/dev/null || true
git remote add github \
"git@github.com:${MIRROR_REPOSITORY}.git"
if [[ "${MIRROR_REF_TYPE}" == "branch" ]]; then
if [[
"${MIRROR_REF_NAME}" != "${MIRROR_BASE_BRANCH}"
]]; then
echo \
"Ignoring non-authoritative branch" \
"${MIRROR_REF_NAME}."
exit 0
fi
git fetch origin \
"+refs/heads/${MIRROR_BASE_BRANCH}:refs/remotes/origin/${MIRROR_BASE_BRANCH}"
local_sha="$(
git rev-parse \
"refs/remotes/origin/${MIRROR_BASE_BRANCH}"
)"
remote_sha="$(
git ls-remote \
github \
"refs/heads/${MIRROR_BASE_BRANCH}" \
| awk 'NR == 1 { print $1 }'
)"
if [[ -z "${remote_sha}" ]]; then
git push github \
"${local_sha}:refs/heads/${MIRROR_BASE_BRANCH}"
exit 0
fi
if [[ "${local_sha}" == "${remote_sha}" ]]; then
echo \
"GitHub ${MIRROR_BASE_BRANCH}" \
"is already synchronized."
exit 0
fi
git fetch \
--no-tags \
github \
"refs/heads/${MIRROR_BASE_BRANCH}:refs/remotes/github/${MIRROR_BASE_BRANCH}"
if ! git merge-base \
--is-ancestor \
"${remote_sha}" \
"${local_sha}"; then
echo \
"Refusing to overwrite GitHub" \
"${MIRROR_BASE_BRANCH}." \
>&2
echo \
"GitHub ${remote_sha} is not an ancestor" \
"of Gitea ${local_sha}." \
>&2
echo \
"Import and resolve the GitHub commit" \
"on Gitea first." \
>&2
exit 1
fi
git push github \
"${local_sha}:refs/heads/${MIRROR_BASE_BRANCH}"
exit 0
fi
if [[ "${MIRROR_REF_TYPE}" == "tag" ]]; then
local_tag_sha="$(
git rev-parse \
"refs/tags/${MIRROR_REF_NAME}"
)"
remote_tag_sha="$(
git ls-remote \
github \
"refs/tags/${MIRROR_REF_NAME}" \
| awk 'NR == 1 { print $1 }'
)"
if [[ -z "${remote_tag_sha}" ]]; then
git push github \
"refs/tags/${MIRROR_REF_NAME}:refs/tags/${MIRROR_REF_NAME}"
exit 0
fi
if [[ "${local_tag_sha}" == "${remote_tag_sha}" ]]; then
echo \
"GitHub tag ${MIRROR_REF_NAME}" \
"is already synchronized."
exit 0
fi
echo \
"Refusing to rewrite GitHub tag" \
"${MIRROR_REF_NAME}." \
>&2
echo \
"GitHub has ${remote_tag_sha};" \
"Gitea has ${local_tag_sha}." \
>&2
exit 1
fi
echo \
"Unsupported ref type: ${MIRROR_REF_TYPE}" \
>&2
exit 1
+336
View File
@@ -0,0 +1,336 @@
name: Sync GitHub PR to Gitea
on:
pull_request_target:
types:
- opened
- reopened
- synchronize
- edited
- ready_for_review
- converted_to_draft
- closed
permissions:
contents: read
pull-requests: read
concurrency:
group: gitea-pr-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
sync:
runs-on: ubuntu-latest
env:
GITEA_URL: ${{ vars.GITEA_URL }}
GITEA_OWNER: ${{ vars.GITEA_OWNER }}
GITEA_REPO: ${{ vars.GITEA_REPO }}
GITEA_SSH_REMOTE: ${{ vars.GITEA_SSH_REMOTE }}
GITEA_BASE_BRANCH: ${{ vars.GITEA_BASE_BRANCH }}
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
GITEA_SSH_KEY: ${{ secrets.GITEA_SSH_KEY }}
GITEA_KNOWN_HOSTS: ${{ secrets.GITEA_KNOWN_HOSTS }}
GH_REPO_TOKEN: ${{ github.token }}
steps:
- name: Synchronize pull request
shell: bash
run: |
set -Eeuo pipefail
RequireVariable() {
local variable_name="$1"
if [[ -z "${!variable_name:-}" ]]; then
echo "Missing required variable: ${variable_name}" >&2
exit 1
fi
}
ApiRequest() {
local method="$1"
local path="$2"
local data="${3:-}"
if [[ -n "${data}" ]]; then
curl \
--silent \
--show-error \
--fail-with-body \
--request "${method}" \
--header "Authorization: token ${GITEA_TOKEN}" \
--header "Content-Type: application/json" \
--data "${data}" \
"${GITEA_URL%/}/api/v1${path}"
else
curl \
--silent \
--show-error \
--fail-with-body \
--request "${method}" \
--header "Authorization: token ${GITEA_TOKEN}" \
--header "Accept: application/json" \
"${GITEA_URL%/}/api/v1${path}"
fi
}
FindGiteaPullRequest() {
local branch_name="$1"
local page
local response
local pull_number
for page in $(seq 1 100); do
response="$(
ApiRequest GET \
"/repos/${GITEA_OWNER}/${GITEA_REPO}/pulls?state=all&page=${page}&limit=50"
)"
pull_number="$(
jq \
-r \
--arg branch_name "${branch_name}" \
--arg base_branch "${GITEA_BASE_BRANCH}" \
'
.[]
| select(
.head.ref == $branch_name
and .base.ref == $base_branch
)
| (.number // .index)
' \
<<<"${response}" \
| head -n 1
)"
if [[
-n "${pull_number}"
&& "${pull_number}" != "null"
]]; then
printf '%s\n' "${pull_number}"
return 0
fi
if [[ "$(jq 'length' <<<"${response}")" -lt 50 ]]; then
break
fi
done
return 1
}
for variable_name in \
GITEA_URL \
GITEA_OWNER \
GITEA_REPO \
GITEA_SSH_REMOTE \
GITEA_BASE_BRANCH \
GITEA_TOKEN \
GITEA_SSH_KEY \
GITEA_KNOWN_HOSTS \
GH_REPO_TOKEN; do
RequireVariable "${variable_name}"
done
action="$(
jq -r '.action' "${GITHUB_EVENT_PATH}"
)"
pr_number="$(
jq -r '.pull_request.number' "${GITHUB_EVENT_PATH}"
)"
pr_title="$(
jq -r '.pull_request.title' "${GITHUB_EVENT_PATH}"
)"
pr_body="$(
jq -r '.pull_request.body // ""' "${GITHUB_EVENT_PATH}"
)"
pr_url="$(
jq -r '.pull_request.html_url' "${GITHUB_EVENT_PATH}"
)"
pr_author="$(
jq -r '.pull_request.user.login' "${GITHUB_EVENT_PATH}"
)"
expected_sha="$(
jq -r '.pull_request.head.sha' "${GITHUB_EVENT_PATH}"
)"
github_base_branch="$(
jq -r '.pull_request.base.ref' "${GITHUB_EVENT_PATH}"
)"
imported_branch="github-pr/${pr_number}"
if [[
"${github_base_branch}" != "${GITEA_BASE_BRANCH}"
]]; then
echo \
"Ignoring PR #${pr_number}: base branch" \
"${github_base_branch} is not ${GITEA_BASE_BRANCH}."
exit 0
fi
install -d -m 700 "${HOME}/.ssh"
printf '%s\n' "${GITEA_SSH_KEY}" \
> "${HOME}/.ssh/id_ed25519"
printf '%s\n' "${GITEA_KNOWN_HOSTS}" \
> "${HOME}/.ssh/known_hosts"
chmod 600 "${HOME}/.ssh/id_ed25519"
chmod 644 "${HOME}/.ssh/known_hosts"
repository_dir="$(mktemp -d)"
trap 'rm -rf "${repository_dir}"' EXIT
git -C "${repository_dir}" init --bare
git -C "${repository_dir}" remote add github \
"https://github.com/${GITHUB_REPOSITORY}.git"
git -C "${repository_dir}" remote add gitea \
"${GITEA_SSH_REMOTE}"
github_basic_auth="$(
printf \
'x-access-token:%s' \
"${GH_REPO_TOKEN}" \
| base64 \
| tr -d '\n'
)"
git -C "${repository_dir}" config \
http.https://github.com/.extraheader \
"AUTHORIZATION: basic ${github_basic_auth}"
gitea_pull_number="$(
FindGiteaPullRequest "${imported_branch}" || true
)"
if [[ "${action}" == "closed" ]]; then
if [[ -n "${gitea_pull_number}" ]]; then
close_payload="$(
jq -n '{state: "closed"}'
)"
ApiRequest \
PATCH \
"/repos/${GITEA_OWNER}/${GITEA_REPO}/pulls/${gitea_pull_number}" \
"${close_payload}" \
>/dev/null
fi
git -C "${repository_dir}" push gitea \
":refs/heads/${imported_branch}" \
|| true
exit 0
fi
git -C "${repository_dir}" fetch \
--no-tags \
github \
"+refs/pull/${pr_number}/head:refs/remotes/github/pr/${pr_number}"
fetched_sha="$(
git -C "${repository_dir}" rev-parse \
"refs/remotes/github/pr/${pr_number}"
)"
if [[ "${fetched_sha}" != "${expected_sha}" ]]; then
echo \
"Fetched ${fetched_sha}, expected ${expected_sha}." \
"Refusing to synchronize." \
>&2
exit 1
fi
current_gitea_sha="$(
git -C "${repository_dir}" ls-remote \
gitea \
"refs/heads/${imported_branch}" \
| awk 'NR == 1 { print $1 }'
)"
if [[ -n "${current_gitea_sha}" ]]; then
git -C "${repository_dir}" push \
--force-with-lease="refs/heads/${imported_branch}:${current_gitea_sha}" \
gitea \
"refs/remotes/github/pr/${pr_number}:refs/heads/${imported_branch}"
else
git -C "${repository_dir}" push \
gitea \
"refs/remotes/github/pr/${pr_number}:refs/heads/${imported_branch}"
fi
marker="<!-- github-pr:${GITHUB_REPOSITORY}#${pr_number} -->"
imported_body="$(cat <<EOF
${marker}
Imported from ${pr_url}.
GitHub author: @${pr_author}
GitHub head commit: \`${expected_sha}\`
---
${pr_body}
EOF
)"
imported_title="[GitHub #${pr_number}] ${pr_title}"
if [[ -z "${gitea_pull_number}" ]]; then
create_payload="$(
jq \
-n \
--arg base "${GITEA_BASE_BRANCH}" \
--arg head "${imported_branch}" \
--arg title "${imported_title}" \
--arg body "${imported_body}" \
'{
base: $base,
head: $head,
title: $title,
body: $body
}'
)"
ApiRequest \
POST \
"/repos/${GITEA_OWNER}/${GITEA_REPO}/pulls" \
"${create_payload}" \
>/dev/null
else
update_payload="$(
jq \
-n \
--arg title "${imported_title}" \
--arg body "${imported_body}" \
'{
title: $title,
body: $body,
state: "open"
}'
)"
ApiRequest \
PATCH \
"/repos/${GITEA_OWNER}/${GITEA_REPO}/pulls/${gitea_pull_number}" \
"${update_payload}" \
>/dev/null
fi
+520 -506
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -512,7 +512,7 @@ fn customFlint(
_: bool, _: bool,
) !void { ) !void {
lib_mod.addImport("intel_c", base_c_mod); lib_mod.addImport("intel_c", base_c_mod);
lib_mod.addImport("shader_compiler", b.createModule(.{ lib_mod.addImport("shader_ir", b.createModule(.{
.root_source_file = b.path("src/compiler/root.zig"), .root_source_file = b.path("src/compiler/root.zig"),
.target = target, .target = target,
.optimize = optimize, .optimize = optimize,
+2
View File
@@ -32,6 +32,7 @@ pub const FlintQueryPool = @import("FlintQueryPool.zig");
pub const FlintRenderPass = @import("FlintRenderPass.zig"); pub const FlintRenderPass = @import("FlintRenderPass.zig");
pub const FlintSampler = @import("FlintSampler.zig"); pub const FlintSampler = @import("FlintSampler.zig");
pub const FlintShaderModule = @import("FlintShaderModule.zig"); pub const FlintShaderModule = @import("FlintShaderModule.zig");
pub const compiler = @import("compiler/root.zig");
pub const Instance = FlintInstance; pub const Instance = FlintInstance;
@@ -89,6 +90,7 @@ test {
std.testing.refAllDecls(FlintRenderPass); std.testing.refAllDecls(FlintRenderPass);
std.testing.refAllDecls(FlintSampler); std.testing.refAllDecls(FlintSampler);
std.testing.refAllDecls(FlintShaderModule); std.testing.refAllDecls(FlintShaderModule);
std.testing.refAllDecls(compiler);
std.testing.refAllDecls(kmd); std.testing.refAllDecls(kmd);
std.testing.refAllDecls(base); std.testing.refAllDecls(base);
} }
+39 -9
View File
@@ -81,9 +81,27 @@ pub fn create(instance: *base.Instance, physical_device: *base.PhysicalDevice, a
const phi_physical_device: *PhiPhysicalDevice = @alignCast(@fieldParentPtr("interface", physical_device)); const phi_physical_device: *PhiPhysicalDevice = @alignCast(@fieldParentPtr("interface", physical_device));
const transport = PhiTransport.init(instance, phi_physical_device.scif_node_id) catch blk: { const transport = PhiTransport.init(instance, phi_physical_device.scif_node_id) catch blk: {
// If first connect failed try to upload the daemon to the card // If the first connection failed, upload and launch the daemon on the card.
try uploadAndLaunchDaemon(instance, allocator, phi_physical_device.mic_device_num); try uploadAndLaunchDaemon(instance, allocator, phi_physical_device.mic_device_num);
break :blk try PhiTransport.init(instance, phi_physical_device.scif_node_id);
const max_connect_attempts = 3;
for (0..max_connect_attempts) |attempt| {
(std.Io.Clock.Duration{
.raw = .fromNanoseconds(std.time.ns_per_s),
.clock = .awake,
}).sleep(instance.io()) catch return VkError.InitializationFailed;
const retry = PhiTransport.init(instance, phi_physical_device.scif_node_id) catch |err| {
if (attempt + 1 == max_connect_attempts) return err;
std.log.scoped(.PhiDevice).debug(
"Phi daemon is not ready; retrying connection ({d}/{d})",
.{ attempt + 1, max_connect_attempts },
);
continue;
};
break :blk retry;
}
unreachable;
}; };
self.* = .{ self.* = .{
@@ -226,10 +244,12 @@ pub fn getDeviceGroupSurfacePresentModesKHR(_: *Interface, _: *base.SurfaceKHR)
fn uploadAndLaunchDaemon(instance: *base.Instance, allocator: std.mem.Allocator, mic_device_num: u32) VkError!void { fn uploadAndLaunchDaemon(instance: *base.Instance, allocator: std.mem.Allocator, mic_device_num: u32) VkError!void {
const io = instance.io(); const io = instance.io();
const process_id = std.os.linux.getpid();
const thread_id = std.Thread.getCurrentId();
const local_path = std.fmt.allocPrint(allocator, "/tmp/ape_phi_device_{d}_{d}.mic", .{ std.os.linux.getpid(), mic_device_num }) catch return VkError.OutOfHostMemory; const local_path = std.fmt.allocPrint(allocator, "/tmp/ape_phi_device_{d}_{d}_{d}.mic", .{ process_id, thread_id, mic_device_num }) catch return VkError.OutOfHostMemory;
defer allocator.free(local_path); defer allocator.free(local_path);
defer std.Io.Dir.deleteFileAbsolute(io, local_path) catch @panic("Caught an error while handling an error"); defer std.Io.Dir.deleteFileAbsolute(io, local_path) catch @panic("Caught an error in a deferred cleanup");
std.Io.Dir.writeFile(.cwd(), io, .{ std.Io.Dir.writeFile(.cwd(), io, .{
.sub_path = local_path, .sub_path = local_path,
@@ -242,19 +262,29 @@ fn uploadAndLaunchDaemon(instance: *base.Instance, allocator: std.mem.Allocator,
const host = std.fmt.allocPrint(allocator, "{s}{d}", .{ config.phi_daemon_host_prefix, mic_device_num }) catch return VkError.OutOfHostMemory; const host = std.fmt.allocPrint(allocator, "{s}{d}", .{ config.phi_daemon_host_prefix, mic_device_num }) catch return VkError.OutOfHostMemory;
defer allocator.free(host); defer allocator.free(host);
const remote_target = std.fmt.allocPrint(allocator, "{s}:{s}", .{ host, config.phi_daemon_remote_path }) catch return VkError.OutOfHostMemory; // Upload beside the daemon and atomically replace it. Copying directly over a
defer allocator.free(remote_target); // running executable fails with ETXTBSY ("Text file busy") on the card.
const remote_upload_path = std.fmt.allocPrint(
allocator,
"{s}.upload-{d}-{d}-{d}",
.{ config.phi_daemon_remote_path, process_id, thread_id, mic_device_num },
) catch return VkError.OutOfHostMemory;
defer allocator.free(remote_upload_path);
const remote_upload_target = std.fmt.allocPrint(allocator, "{s}:{s}", .{ host, remote_upload_path }) catch return VkError.OutOfHostMemory;
defer allocator.free(remote_upload_target);
std.log.scoped(.PhiDevice).debug("Uploading Phi daemon to {s}", .{remote_upload_target});
try runHostCommand(instance, allocator, &.{ try runHostCommand(instance, allocator, &.{
"scp", "scp",
local_path, local_path,
remote_target, remote_upload_target,
}); });
const launch_command = std.fmt.allocPrint( const launch_command = std.fmt.allocPrint(
allocator, allocator,
"chmod +x {s} && nohup {s} >/tmp/phi_device.log 2>&1 </dev/null &", "chmod +x {s} && mv -f {s} {s} && (nohup {s} >/tmp/phi_device.log 2>&1 </dev/null &)",
.{ config.phi_daemon_remote_path, config.phi_daemon_remote_path }, .{ remote_upload_path, remote_upload_path, config.phi_daemon_remote_path, config.phi_daemon_remote_path },
) catch return VkError.OutOfHostMemory; ) catch return VkError.OutOfHostMemory;
defer allocator.free(launch_command); defer allocator.free(launch_command);
+5
View File
@@ -45,6 +45,11 @@ pub fn init(instance: *base.Instance, node_id: u16) VkError!Self {
} }
pub fn deinit(self: *Self) void { pub fn deinit(self: *Self) void {
var reply: proto.PhiResult = undefined;
self.request(proto.PHI_PACKET_SHUTDOWN, &.{}, std.mem.asBytes(&reply)) catch |err| {
std.log.scoped(.PhiTransport).warn("Failed to shut down remote session: {s}", .{@errorName(err)});
};
_ = scif.close(self.epd); _ = scif.close(self.epd);
scif.unload(); scif.unload();
std.log.scoped(.PhiTransport).info("Closed connection", .{}); std.log.scoped(.PhiTransport).info("Closed connection", .{});
+2 -2
View File
@@ -1,5 +1,5 @@
#include <Daemon.h>
#include <CommandBuffer.h> #include <CommandBuffer.h>
#include <Daemon.h>
#include <Logger.h> #include <Logger.h>
#include <Memory.h> #include <Memory.h>
@@ -50,7 +50,7 @@ scif_epd_t StartDaemon()
return 0; return 0;
} }
if(scif_listen(endpoint, 1) < 0) if(scif_listen(endpoint, 16) < 0)
{ {
PhiLogError("Could not listen to SCIF port"); PhiLogError("Could not listen to SCIF port");
scif_close(endpoint); scif_close(endpoint);
+1 -1
View File
@@ -25,7 +25,7 @@ int HandleAllocMemory(scif_epd_t endpoint, const PhiMessageHeader* header)
if(ReadAll(endpoint, &request, sizeof(request)) < 0) if(ReadAll(endpoint, &request, sizeof(request)) < 0)
return -1; return -1;
void* memory = malloc((size_t)request.size); const void* memory = malloc((size_t)request.size);
if(memory == NULL) if(memory == NULL)
{ {
+29 -3
View File
@@ -1,13 +1,35 @@
#include <errno.h>
#include <pthread.h>
#include <stdint.h>
#include <Daemon.h> #include <Daemon.h>
#include <Logger.h> #include <Logger.h>
static void* HandleClient(void* const argument)
{
scif_epd_t client = (scif_epd_t)(intptr_t)argument;
(void)HandlePacket(client);
scif_close(client);
return NULL;
}
int main(void) int main(void)
{ {
scif_epd_t endpoint = StartDaemon(); scif_epd_t endpoint = StartDaemon();
pthread_attr_t client_thread_attributes;
if(endpoint == 0) if(endpoint == 0)
return 1; return 1;
if(pthread_attr_init(&client_thread_attributes) != 0 ||
pthread_attr_setdetachstate(&client_thread_attributes, PTHREAD_CREATE_DETACHED) != 0)
{
PhiLogError("Could not initialize client thread attributes");
ShutdownDaemon(endpoint);
return 1;
}
for(;;) for(;;)
{ {
struct scif_portID peer; struct scif_portID peer;
@@ -21,11 +43,15 @@ int main(void)
break; break;
} }
(void)HandlePacket(client); pthread_t client_thread;
if(pthread_create(&client_thread, &client_thread_attributes, HandleClient, (void*)(intptr_t)client) != 0)
scif_close(client); {
PhiLogError("Could not create SCIF client thread");
scif_close(client);
}
} }
pthread_attr_destroy(&client_thread_attributes);
ShutdownDaemon(endpoint); ShutdownDaemon(endpoint);
return 0; return 0;
+1 -1
View File
@@ -1,7 +1,7 @@
#ifndef APE_PHI_PROTOCOL_H #ifndef APE_PHI_PROTOCOL_H
#define APE_PHI_PROTOCOL_H #define APE_PHI_PROTOCOL_H
#include "Commands.h" #include "Commands.h" // IWYU pragma: keep
#include <stdint.h> #include <stdint.h>
#define PHI_PROTOCOL_MAGIC 0x50484941u #define PHI_PROTOCOL_MAGIC 0x50484941u