ci: retry transient QuickStack rollouts #1
@@ -134,6 +134,26 @@ async function trigger(client, appId, sleep) {
|
||||
return { deploymentId, deployment: await waitForDeployment(client, appId, deploymentId, sleep) };
|
||||
}
|
||||
|
||||
async function triggerWithRetries({
|
||||
client, appId, sleep, attempts = 1, retryDelayMs = 15_000, label = "deployment", restoreConfig,
|
||||
}) {
|
||||
let latestError;
|
||||
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
||||
try {
|
||||
return await trigger(client, appId, sleep);
|
||||
} catch (error) {
|
||||
latestError = error;
|
||||
if (attempt >= attempts) throw error;
|
||||
console.log(
|
||||
`QuickStack ${label} attempt ${attempt}/${attempts} failed; retrying in ${retryDelayMs / 1000}s: ${error instanceof Error ? error.message : error}`,
|
||||
);
|
||||
await sleep(retryDelayMs);
|
||||
await restoreConfig();
|
||||
}
|
||||
}
|
||||
throw latestError;
|
||||
}
|
||||
|
||||
export async function verifyEndpoint(postflight, { sha, fetchImpl = fetch, sleep }) {
|
||||
if (!postflight) return null;
|
||||
const url = requiredString(postflight.url, "Postflight URL");
|
||||
@@ -169,10 +189,17 @@ export async function verifyEndpoint(postflight, { sha, fetchImpl = fetch, sleep
|
||||
|
||||
export async function deployExactImage({
|
||||
client, appId, image, registryUsername, registryToken, environment = {},
|
||||
healthCheckTcpPort, postflight, sha, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
healthCheckTcpPort, postflight, sha, deploymentAttempts = 1, deploymentRetrySeconds = 15,
|
||||
sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
fetchImpl = fetch,
|
||||
}) {
|
||||
if (!/@sha256:[0-9a-f]{64}$/.test(image)) throw new Error("Deployment image must use an exact sha256 digest.");
|
||||
const attempts = Number(deploymentAttempts);
|
||||
const retryDelaySeconds = Number(deploymentRetrySeconds);
|
||||
if (!Number.isInteger(attempts) || attempts < 1) throw new Error("Deployment attempts must be a positive integer.");
|
||||
if (!Number.isFinite(retryDelaySeconds) || retryDelaySeconds < 0) {
|
||||
throw new Error("Deployment retry seconds must be a non-negative number.");
|
||||
}
|
||||
const current = await client.getApp(appId);
|
||||
const previous = toSavePayload(current);
|
||||
const next = {
|
||||
@@ -186,7 +213,15 @@ export async function deployExactImage({
|
||||
};
|
||||
await client.saveApp(next);
|
||||
try {
|
||||
const result = await trigger(client, appId, sleep);
|
||||
const result = await triggerWithRetries({
|
||||
client,
|
||||
appId,
|
||||
sleep,
|
||||
attempts,
|
||||
retryDelayMs: retryDelaySeconds * 1000,
|
||||
label: "deployment",
|
||||
restoreConfig: () => client.saveApp(next),
|
||||
});
|
||||
const persisted = await client.getApp(appId);
|
||||
if (persisted.sourceType !== "CONTAINER" || persisted.containerImageSource !== image) {
|
||||
throw new Error("QuickStack did not persist the exact image digest.");
|
||||
@@ -196,7 +231,15 @@ export async function deployExactImage({
|
||||
} catch (error) {
|
||||
await client.saveApp(previous);
|
||||
try {
|
||||
const rollback = await trigger(client, appId, sleep);
|
||||
const rollback = await triggerWithRetries({
|
||||
client,
|
||||
appId,
|
||||
sleep,
|
||||
attempts,
|
||||
retryDelayMs: retryDelaySeconds * 1000,
|
||||
label: "rollback",
|
||||
restoreConfig: () => client.saveApp(previous),
|
||||
});
|
||||
throw new Error(
|
||||
`Deployment failed and was rolled back with ${rollback.deploymentId}: ${error instanceof Error ? error.message : error}`,
|
||||
);
|
||||
@@ -249,11 +292,23 @@ export function validateTarget(target) {
|
||||
const image = requiredString(target.image, `Image for ${name}`).toLowerCase();
|
||||
if (!/^[a-z0-9][a-z0-9._/-]*$/.test(image)) throw new Error(`Invalid OCI image path ${image}.`);
|
||||
const appId = requiredString(target.appId, `QuickStack app ID for ${name}`);
|
||||
const deploymentAttempts = target.deploymentAttempts === undefined ? undefined : Number(target.deploymentAttempts);
|
||||
const deploymentRetrySeconds = target.deploymentRetrySeconds === undefined
|
||||
? undefined
|
||||
: Number(target.deploymentRetrySeconds);
|
||||
if (deploymentAttempts !== undefined && (!Number.isInteger(deploymentAttempts) || deploymentAttempts < 1)) {
|
||||
throw new Error(`Deployment attempts for ${name} must be a positive integer.`);
|
||||
}
|
||||
if (deploymentRetrySeconds !== undefined && (!Number.isFinite(deploymentRetrySeconds) || deploymentRetrySeconds < 0)) {
|
||||
throw new Error(`Deployment retry seconds for ${name} must be a non-negative number.`);
|
||||
}
|
||||
return {
|
||||
...target,
|
||||
name,
|
||||
image,
|
||||
appId,
|
||||
...(deploymentAttempts === undefined ? {} : { deploymentAttempts }),
|
||||
...(deploymentRetrySeconds === undefined ? {} : { deploymentRetrySeconds }),
|
||||
dockerfile: safeRelative(target.dockerfile ?? "Dockerfile", `Dockerfile for ${name}`),
|
||||
context: safeRelative(target.context ?? ".", `Build context for ${name}`),
|
||||
};
|
||||
@@ -359,6 +414,8 @@ async function main() {
|
||||
environment: target.environment,
|
||||
healthCheckTcpPort: target.healthCheckTcpPort,
|
||||
postflight: target.postflight,
|
||||
deploymentAttempts: target.deploymentAttempts,
|
||||
deploymentRetrySeconds: target.deploymentRetrySeconds,
|
||||
sha,
|
||||
});
|
||||
console.log(`Deployed ${target.name}: ${result.image}`);
|
||||
|
||||
@@ -59,6 +59,10 @@ test("resolves push and pull request deployment branches without accepting empty
|
||||
test("validates OCI paths and repository-local build paths", () => {
|
||||
assert.equal(validateTarget({ name: "Web", image: "Owner/Web", appId: "app-1" }).image, "owner/web");
|
||||
assert.throws(() => validateTarget({ name: "Web", image: "owner/web", appId: "app-1", context: "../secret" }), /inside/);
|
||||
assert.throws(
|
||||
() => validateTarget({ name: "Web", image: "owner/web", appId: "app-1", deploymentAttempts: 0 }),
|
||||
/positive integer/,
|
||||
);
|
||||
});
|
||||
|
||||
test("preserves response-only fields and existing environment secrets safely", () => {
|
||||
@@ -111,3 +115,35 @@ test("failed public verification restores and redeploys the previous source", as
|
||||
assert.equal(current.sourceType, original.sourceType);
|
||||
assert.equal(current.containerImageSource, original.containerImageSource);
|
||||
});
|
||||
|
||||
test("transient QuickStack deployment failures retry the desired immutable image", async () => {
|
||||
let current = app();
|
||||
let deployments = 0;
|
||||
const image = `gitea.example/owner/app@sha256:${"b".repeat(64)}`;
|
||||
const client = {
|
||||
getApp: async () => current,
|
||||
saveApp: async (payload) => { current = { ...current, ...payload }; },
|
||||
deployApp: async () => ({ deploymentId: `deployment-${++deployments}` }),
|
||||
getDeployment: async (_appId, deploymentId) => ({
|
||||
deploymentId,
|
||||
status: deploymentId === "deployment-1" ? "ERROR" : "DEPLOYED",
|
||||
}),
|
||||
};
|
||||
|
||||
const result = await deployExactImage({
|
||||
client,
|
||||
appId: "app-one",
|
||||
image,
|
||||
registryUsername: "deploy",
|
||||
registryToken: "token",
|
||||
sha: "2".repeat(40),
|
||||
deploymentAttempts: 2,
|
||||
deploymentRetrySeconds: 0,
|
||||
sleep: async () => {},
|
||||
});
|
||||
|
||||
assert.equal(deployments, 2);
|
||||
assert.equal(result.image, image);
|
||||
assert.equal(current.sourceType, "CONTAINER");
|
||||
assert.equal(current.containerImageSource, image);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
name: Platform CI tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: QuickStack deploy action tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Run action tests
|
||||
run: node --test .gitea/actions/quickstack-oci/deploy.test.mjs
|
||||
|
||||
Reference in New Issue
Block a user