ci: retry transient QuickStack rollouts

.gitea/actions/quickstack-oci/deploy.mjs

@@ -134,6 +134,26 @@ async function trigger(client, appId, sleep) {
   return { deploymentId, deployment: await waitForDeployment(client, appId, deploymentId, sleep) };
 }
 
+async function triggerWithRetries({
+  client, appId, sleep, attempts = 1, retryDelayMs = 15_000, label = "deployment", restoreConfig,
+}) {
+  let latestError;
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    try {
+      return await trigger(client, appId, sleep);
+    } catch (error) {
+      latestError = error;
+      if (attempt >= attempts) throw error;
+      console.log(
+        `QuickStack ${label} attempt ${attempt}/${attempts} failed; retrying in ${retryDelayMs / 1000}s: ${error instanceof Error ? error.message : error}`,
+      );
+      await sleep(retryDelayMs);
+      await restoreConfig();
+    }
+  }
+  throw latestError;
+}
+
 export async function verifyEndpoint(postflight, { sha, fetchImpl = fetch, sleep }) {
   if (!postflight) return null;
   const url = requiredString(postflight.url, "Postflight URL");
@@ -169,10 +189,17 @@ export async function verifyEndpoint(postflight, { sha, fetchImpl = fetch, sleep
 
 export async function deployExactImage({
   client, appId, image, registryUsername, registryToken, environment = {},
-  healthCheckTcpPort, postflight, sha, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+  healthCheckTcpPort, postflight, sha, deploymentAttempts = 1, deploymentRetrySeconds = 15,
+  sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
   fetchImpl = fetch,
 }) {
   if (!/@sha256:[0-9a-f]{64}$/.test(image)) throw new Error("Deployment image must use an exact sha256 digest.");
+  const attempts = Number(deploymentAttempts);
+  const retryDelaySeconds = Number(deploymentRetrySeconds);
+  if (!Number.isInteger(attempts) || attempts < 1) throw new Error("Deployment attempts must be a positive integer.");
+  if (!Number.isFinite(retryDelaySeconds) || retryDelaySeconds < 0) {
+    throw new Error("Deployment retry seconds must be a non-negative number.");
+  }
   const current = await client.getApp(appId);
   const previous = toSavePayload(current);
   const next = {
@@ -186,7 +213,15 @@ export async function deployExactImage({
   };
   await client.saveApp(next);
   try {
-    const result = await trigger(client, appId, sleep);
+    const result = await triggerWithRetries({
+      client,
+      appId,
+      sleep,
+      attempts,
+      retryDelayMs: retryDelaySeconds * 1000,
+      label: "deployment",
+      restoreConfig: () => client.saveApp(next),
+    });
     const persisted = await client.getApp(appId);
     if (persisted.sourceType !== "CONTAINER" || persisted.containerImageSource !== image) {
       throw new Error("QuickStack did not persist the exact image digest.");
@@ -196,7 +231,15 @@ export async function deployExactImage({
   } catch (error) {
     await client.saveApp(previous);
     try {
-      const rollback = await trigger(client, appId, sleep);
+      const rollback = await triggerWithRetries({
+        client,
+        appId,
+        sleep,
+        attempts,
+        retryDelayMs: retryDelaySeconds * 1000,
+        label: "rollback",
+        restoreConfig: () => client.saveApp(previous),
+      });
       throw new Error(
         `Deployment failed and was rolled back with ${rollback.deploymentId}: ${error instanceof Error ? error.message : error}`,
       );
@@ -249,11 +292,23 @@ export function validateTarget(target) {
   const image = requiredString(target.image, `Image for ${name}`).toLowerCase();
   if (!/^[a-z0-9][a-z0-9._/-]*$/.test(image)) throw new Error(`Invalid OCI image path ${image}.`);
   const appId = requiredString(target.appId, `QuickStack app ID for ${name}`);
+  const deploymentAttempts = target.deploymentAttempts === undefined ? undefined : Number(target.deploymentAttempts);
+  const deploymentRetrySeconds = target.deploymentRetrySeconds === undefined
+    ? undefined
+    : Number(target.deploymentRetrySeconds);
+  if (deploymentAttempts !== undefined && (!Number.isInteger(deploymentAttempts) || deploymentAttempts < 1)) {
+    throw new Error(`Deployment attempts for ${name} must be a positive integer.`);
+  }
+  if (deploymentRetrySeconds !== undefined && (!Number.isFinite(deploymentRetrySeconds) || deploymentRetrySeconds < 0)) {
+    throw new Error(`Deployment retry seconds for ${name} must be a non-negative number.`);
+  }
   return {
     ...target,
     name,
     image,
     appId,
+    ...(deploymentAttempts === undefined ? {} : { deploymentAttempts }),
+    ...(deploymentRetrySeconds === undefined ? {} : { deploymentRetrySeconds }),
     dockerfile: safeRelative(target.dockerfile ?? "Dockerfile", `Dockerfile for ${name}`),
     context: safeRelative(target.context ?? ".", `Build context for ${name}`),
   };
@@ -359,6 +414,8 @@ async function main() {
         environment: target.environment,
         healthCheckTcpPort: target.healthCheckTcpPort,
         postflight: target.postflight,
+        deploymentAttempts: target.deploymentAttempts,
+        deploymentRetrySeconds: target.deploymentRetrySeconds,
         sha,
       });
       console.log(`Deployed ${target.name}: ${result.image}`);