From 09a9e6884795d568380a5be11730b049acbbc69e Mon Sep 17 00:00:00 2001 From: Vadime Date: Tue, 18 Aug 2026 19:30:57 +0200 Subject: [PATCH] ci: retry transient QuickStack rollouts --- .gitea/actions/quickstack-oci/deploy.mjs | 63 ++++++++++++++++++- .gitea/actions/quickstack-oci/deploy.test.mjs | 36 +++++++++++ 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/.gitea/actions/quickstack-oci/deploy.mjs b/.gitea/actions/quickstack-oci/deploy.mjs index c2671c6..1748658 100644 --- a/.gitea/actions/quickstack-oci/deploy.mjs +++ b/.gitea/actions/quickstack-oci/deploy.mjs @@ -134,6 +134,26 @@ async function trigger(client, appId, sleep) { return { deploymentId, deployment: await waitForDeployment(client, appId, deploymentId, sleep) }; } +async function triggerWithRetries({ + client, appId, sleep, attempts = 1, retryDelayMs = 15_000, label = "deployment", restoreConfig, +}) { + let latestError; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + return await trigger(client, appId, sleep); + } catch (error) { + latestError = error; + if (attempt >= attempts) throw error; + console.log( + `QuickStack ${label} attempt ${attempt}/${attempts} failed; retrying in ${retryDelayMs / 1000}s: ${error instanceof Error ? error.message : error}`, + ); + await sleep(retryDelayMs); + await restoreConfig(); + } + } + throw latestError; +} + export async function verifyEndpoint(postflight, { sha, fetchImpl = fetch, sleep }) { if (!postflight) return null; const url = requiredString(postflight.url, "Postflight URL"); @@ -169,10 +189,17 @@ export async function verifyEndpoint(postflight, { sha, fetchImpl = fetch, sleep export async function deployExactImage({ client, appId, image, registryUsername, registryToken, environment = {}, - healthCheckTcpPort, postflight, sha, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + healthCheckTcpPort, postflight, sha, deploymentAttempts = 1, deploymentRetrySeconds = 15, + sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), fetchImpl = fetch, }) { if (!/@sha256:[0-9a-f]{64}$/.test(image)) throw new Error("Deployment image must use an exact sha256 digest."); + const attempts = Number(deploymentAttempts); + const retryDelaySeconds = Number(deploymentRetrySeconds); + if (!Number.isInteger(attempts) || attempts < 1) throw new Error("Deployment attempts must be a positive integer."); + if (!Number.isFinite(retryDelaySeconds) || retryDelaySeconds < 0) { + throw new Error("Deployment retry seconds must be a non-negative number."); + } const current = await client.getApp(appId); const previous = toSavePayload(current); const next = { @@ -186,7 +213,15 @@ export async function deployExactImage({ }; await client.saveApp(next); try { - const result = await trigger(client, appId, sleep); + const result = await triggerWithRetries({ + client, + appId, + sleep, + attempts, + retryDelayMs: retryDelaySeconds * 1000, + label: "deployment", + restoreConfig: () => client.saveApp(next), + }); const persisted = await client.getApp(appId); if (persisted.sourceType !== "CONTAINER" || persisted.containerImageSource !== image) { throw new Error("QuickStack did not persist the exact image digest."); @@ -196,7 +231,15 @@ export async function deployExactImage({ } catch (error) { await client.saveApp(previous); try { - const rollback = await trigger(client, appId, sleep); + const rollback = await triggerWithRetries({ + client, + appId, + sleep, + attempts, + retryDelayMs: retryDelaySeconds * 1000, + label: "rollback", + restoreConfig: () => client.saveApp(previous), + }); throw new Error( `Deployment failed and was rolled back with ${rollback.deploymentId}: ${error instanceof Error ? error.message : error}`, ); @@ -249,11 +292,23 @@ export function validateTarget(target) { const image = requiredString(target.image, `Image for ${name}`).toLowerCase(); if (!/^[a-z0-9][a-z0-9._/-]*$/.test(image)) throw new Error(`Invalid OCI image path ${image}.`); const appId = requiredString(target.appId, `QuickStack app ID for ${name}`); + const deploymentAttempts = target.deploymentAttempts === undefined ? undefined : Number(target.deploymentAttempts); + const deploymentRetrySeconds = target.deploymentRetrySeconds === undefined + ? undefined + : Number(target.deploymentRetrySeconds); + if (deploymentAttempts !== undefined && (!Number.isInteger(deploymentAttempts) || deploymentAttempts < 1)) { + throw new Error(`Deployment attempts for ${name} must be a positive integer.`); + } + if (deploymentRetrySeconds !== undefined && (!Number.isFinite(deploymentRetrySeconds) || deploymentRetrySeconds < 0)) { + throw new Error(`Deployment retry seconds for ${name} must be a non-negative number.`); + } return { ...target, name, image, appId, + ...(deploymentAttempts === undefined ? {} : { deploymentAttempts }), + ...(deploymentRetrySeconds === undefined ? {} : { deploymentRetrySeconds }), dockerfile: safeRelative(target.dockerfile ?? "Dockerfile", `Dockerfile for ${name}`), context: safeRelative(target.context ?? ".", `Build context for ${name}`), }; @@ -359,6 +414,8 @@ async function main() { environment: target.environment, healthCheckTcpPort: target.healthCheckTcpPort, postflight: target.postflight, + deploymentAttempts: target.deploymentAttempts, + deploymentRetrySeconds: target.deploymentRetrySeconds, sha, }); console.log(`Deployed ${target.name}: ${result.image}`); diff --git a/.gitea/actions/quickstack-oci/deploy.test.mjs b/.gitea/actions/quickstack-oci/deploy.test.mjs index 2dcdc6f..bb17127 100644 --- a/.gitea/actions/quickstack-oci/deploy.test.mjs +++ b/.gitea/actions/quickstack-oci/deploy.test.mjs @@ -59,6 +59,10 @@ test("resolves push and pull request deployment branches without accepting empty test("validates OCI paths and repository-local build paths", () => { assert.equal(validateTarget({ name: "Web", image: "Owner/Web", appId: "app-1" }).image, "owner/web"); assert.throws(() => validateTarget({ name: "Web", image: "owner/web", appId: "app-1", context: "../secret" }), /inside/); + assert.throws( + () => validateTarget({ name: "Web", image: "owner/web", appId: "app-1", deploymentAttempts: 0 }), + /positive integer/, + ); }); test("preserves response-only fields and existing environment secrets safely", () => { @@ -111,3 +115,35 @@ test("failed public verification restores and redeploys the previous source", as assert.equal(current.sourceType, original.sourceType); assert.equal(current.containerImageSource, original.containerImageSource); }); + +test("transient QuickStack deployment failures retry the desired immutable image", async () => { + let current = app(); + let deployments = 0; + const image = `gitea.example/owner/app@sha256:${"b".repeat(64)}`; + const client = { + getApp: async () => current, + saveApp: async (payload) => { current = { ...current, ...payload }; }, + deployApp: async () => ({ deploymentId: `deployment-${++deployments}` }), + getDeployment: async (_appId, deploymentId) => ({ + deploymentId, + status: deploymentId === "deployment-1" ? "ERROR" : "DEPLOYED", + }), + }; + + const result = await deployExactImage({ + client, + appId: "app-one", + image, + registryUsername: "deploy", + registryToken: "token", + sha: "2".repeat(40), + deploymentAttempts: 2, + deploymentRetrySeconds: 0, + sleep: async () => {}, + }); + + assert.equal(deployments, 2); + assert.equal(result.image, image); + assert.equal(current.sourceType, "CONTAINER"); + assert.equal(current.containerImageSource, image); +});