ci: retry transient QuickStack rollouts

.gitea/actions/quickstack-oci/deploy.mjs

@@ -134,6 +134,26 @@ async function trigger(client, appId, sleep) {
   return { deploymentId, deployment: await waitForDeployment(client, appId, deploymentId, sleep) };
 }
 
+async function triggerWithRetries({
+  client, appId, sleep, attempts = 1, retryDelayMs = 15_000, label = "deployment", restoreConfig,
+}) {
+  let latestError;
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    try {
+      return await trigger(client, appId, sleep);
+    } catch (error) {
+      latestError = error;
+      if (attempt >= attempts) throw error;
+      console.log(
+        `QuickStack ${label} attempt ${attempt}/${attempts} failed; retrying in ${retryDelayMs / 1000}s: ${error instanceof Error ? error.message : error}`,
+      );
+      await sleep(retryDelayMs);
+      await restoreConfig();
+    }
+  }
+  throw latestError;
+}
+
 export async function verifyEndpoint(postflight, { sha, fetchImpl = fetch, sleep }) {
   if (!postflight) return null;
   const url = requiredString(postflight.url, "Postflight URL");
@@ -169,10 +189,17 @@ export async function verifyEndpoint(postflight, { sha, fetchImpl = fetch, sleep
 
 export async function deployExactImage({
   client, appId, image, registryUsername, registryToken, environment = {},
-  healthCheckTcpPort, postflight, sha, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+  healthCheckTcpPort, postflight, sha, deploymentAttempts = 1, deploymentRetrySeconds = 15,
+  sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
   fetchImpl = fetch,
 }) {
   if (!/@sha256:[0-9a-f]{64}$/.test(image)) throw new Error("Deployment image must use an exact sha256 digest.");
+  const attempts = Number(deploymentAttempts);
+  const retryDelaySeconds = Number(deploymentRetrySeconds);
+  if (!Number.isInteger(attempts) || attempts < 1) throw new Error("Deployment attempts must be a positive integer.");
+  if (!Number.isFinite(retryDelaySeconds) || retryDelaySeconds < 0) {
+    throw new Error("Deployment retry seconds must be a non-negative number.");
+  }
   const current = await client.getApp(appId);
   const previous = toSavePayload(current);
   const next = {
@@ -186,7 +213,15 @@ export async function deployExactImage({
   };
   await client.saveApp(next);
   try {
-    const result = await trigger(client, appId, sleep);
+    const result = await triggerWithRetries({
+      client,
+      appId,
+      sleep,
+      attempts,
+      retryDelayMs: retryDelaySeconds * 1000,
+      label: "deployment",
+      restoreConfig: () => client.saveApp(next),
+    });
     const persisted = await client.getApp(appId);
     if (persisted.sourceType !== "CONTAINER" || persisted.containerImageSource !== image) {
       throw new Error("QuickStack did not persist the exact image digest.");
@@ -196,7 +231,15 @@ export async function deployExactImage({
   } catch (error) {
     await client.saveApp(previous);
     try {
-      const rollback = await trigger(client, appId, sleep);
+      const rollback = await triggerWithRetries({
+        client,
+        appId,
+        sleep,
+        attempts,
+        retryDelayMs: retryDelaySeconds * 1000,
+        label: "rollback",
+        restoreConfig: () => client.saveApp(previous),
+      });
       throw new Error(
         `Deployment failed and was rolled back with ${rollback.deploymentId}: ${error instanceof Error ? error.message : error}`,
       );
@@ -249,11 +292,23 @@ export function validateTarget(target) {
   const image = requiredString(target.image, `Image for ${name}`).toLowerCase();
   if (!/^[a-z0-9][a-z0-9._/-]*$/.test(image)) throw new Error(`Invalid OCI image path ${image}.`);
   const appId = requiredString(target.appId, `QuickStack app ID for ${name}`);
+  const deploymentAttempts = target.deploymentAttempts === undefined ? undefined : Number(target.deploymentAttempts);
+  const deploymentRetrySeconds = target.deploymentRetrySeconds === undefined
+    ? undefined
+    : Number(target.deploymentRetrySeconds);
+  if (deploymentAttempts !== undefined && (!Number.isInteger(deploymentAttempts) || deploymentAttempts < 1)) {
+    throw new Error(`Deployment attempts for ${name} must be a positive integer.`);
+  }
+  if (deploymentRetrySeconds !== undefined && (!Number.isFinite(deploymentRetrySeconds) || deploymentRetrySeconds < 0)) {
+    throw new Error(`Deployment retry seconds for ${name} must be a non-negative number.`);
+  }
   return {
     ...target,
     name,
     image,
     appId,
+    ...(deploymentAttempts === undefined ? {} : { deploymentAttempts }),
+    ...(deploymentRetrySeconds === undefined ? {} : { deploymentRetrySeconds }),
     dockerfile: safeRelative(target.dockerfile ?? "Dockerfile", `Dockerfile for ${name}`),
     context: safeRelative(target.context ?? ".", `Build context for ${name}`),
   };
@@ -359,6 +414,8 @@ async function main() {
         environment: target.environment,
         healthCheckTcpPort: target.healthCheckTcpPort,
         postflight: target.postflight,
+        deploymentAttempts: target.deploymentAttempts,
+        deploymentRetrySeconds: target.deploymentRetrySeconds,
         sha,
       });
       console.log(`Deployed ${target.name}: ${result.image}`);

.gitea/actions/quickstack-oci/deploy.test.mjs

@@ -59,6 +59,10 @@ test("resolves push and pull request deployment branches without accepting empty
 test("validates OCI paths and repository-local build paths", () => {
   assert.equal(validateTarget({ name: "Web", image: "Owner/Web", appId: "app-1" }).image, "owner/web");
   assert.throws(() => validateTarget({ name: "Web", image: "owner/web", appId: "app-1", context: "../secret" }), /inside/);
+  assert.throws(
+    () => validateTarget({ name: "Web", image: "owner/web", appId: "app-1", deploymentAttempts: 0 }),
+    /positive integer/,
+  );
 });
 
 test("preserves response-only fields and existing environment secrets safely", () => {
@@ -111,3 +115,35 @@ test("failed public verification restores and redeploys the previous source", as
   assert.equal(current.sourceType, original.sourceType);
   assert.equal(current.containerImageSource, original.containerImageSource);
 });
+
+test("transient QuickStack deployment failures retry the desired immutable image", async () => {
+  let current = app();
+  let deployments = 0;
+  const image = `gitea.example/owner/app@sha256:${"b".repeat(64)}`;
+  const client = {
+    getApp: async () => current,
+    saveApp: async (payload) => { current = { ...current, ...payload }; },
+    deployApp: async () => ({ deploymentId: `deployment-${++deployments}` }),
+    getDeployment: async (_appId, deploymentId) => ({
+      deploymentId,
+      status: deploymentId === "deployment-1" ? "ERROR" : "DEPLOYED",
+    }),
+  };
+
+  const result = await deployExactImage({
+    client,
+    appId: "app-one",
+    image,
+    registryUsername: "deploy",
+    registryToken: "token",
+    sha: "2".repeat(40),
+    deploymentAttempts: 2,
+    deploymentRetrySeconds: 0,
+    sleep: async () => {},
+  });
+
+  assert.equal(deployments, 2);
+  assert.equal(result.image, image);
+  assert.equal(current.sourceType, "CONTAINER");
+  assert.equal(current.containerImageSource, image);
+});