From 9bc520ce64badf2a8777ec34ff0375df7ee8d39a Mon Sep 17 00:00:00 2001 From: Vadime Date: Tue, 25 Aug 2026 15:59:41 +0200 Subject: [PATCH] ci: parallelize cached artifact delivery --- .gitea/actions/quickstack-oci/deploy.mjs | 160 ++++++++++++++++------- 1 file changed, 113 insertions(+), 47 deletions(-) diff --git a/.gitea/actions/quickstack-oci/deploy.mjs b/.gitea/actions/quickstack-oci/deploy.mjs index 2d6835c..e567cd6 100644 --- a/.gitea/actions/quickstack-oci/deploy.mjs +++ b/.gitea/actions/quickstack-oci/deploy.mjs @@ -541,6 +541,27 @@ export function resolvePullRequestHeadBranch({ environment = process.env, eventP return requiredString(payload?.pull_request?.head?.ref, "Pull request head branch"); } +export function isSameRepositoryPullRequest({ eventName, environment = process.env, eventPayload } = {}) { + if (eventName !== "pull_request") return false; + + let payload = eventPayload; + if (payload === undefined) { + const eventPath = String(environment.GITHUB_EVENT_PATH || environment.GITEA_EVENT_PATH || "").trim(); + if (eventPath) { + payload = JSON.parse(fs.readFileSync(eventPath, "utf8")); + } + } + const baseRepository = String( + environment.GITHUB_REPOSITORY || + environment.GITEA_REPOSITORY || + payload?.repository?.full_name || + payload?.pull_request?.base?.repo?.full_name || + "", + ).trim(); + const headRepository = String(payload?.pull_request?.head?.repo?.full_name || "").trim(); + return Boolean(baseRepository && headRepository && baseRepository === headRepository); +} + export function classifyVersion2Execution(pipeline, eventName) { if (eventName === "pull_request") { return pipeline.strategy === "promote" ? "validate-candidate" : "build-validation"; @@ -586,6 +607,24 @@ function dockerLogin(registry, dockerEnv) { ); } +function ensureBuildxBuilder(dockerEnv) { + const builder = "quickstack-registry-cache"; + const existing = spawnSync( + "docker", + ["buildx", "inspect", builder], + { env: dockerEnv, encoding: "utf8", stdio: "ignore" }, + ); + if (existing.status !== 0) { + run( + "docker", + ["buildx", "create", "--name", builder, "--driver", "docker-container"], + { env: dockerEnv }, + ); + } + run("docker", ["buildx", "inspect", "--bootstrap", builder], { env: dockerEnv }); + return builder; +} + function exactDigestFromPush(pushOutput, taggedImage) { const digest = [...pushOutput.matchAll(/digest:\s*(sha256:[0-9a-f]{64})/g)].at(-1)?.[1]; if (!digest) throw new Error(`Registry did not return a digest for ${taggedImage}.`); @@ -598,11 +637,20 @@ function verifyRequiredContainerFiles(taggedImage, requiredFiles, options) { } } -function buildArtifact({ artifact, registry, sha, workspace, dockerEnv, validationOnly }) { +function buildArtifact({ + artifact, + registry, + sha, + workspace, + dockerEnv, + validationOnly, + useRegistryCache = !validationOnly, +}) { const taggedImage = `${registry}/${artifact.image}:sha-${sha}`; const cacheImage = `${registry}/${artifact.image}:buildcache`; + const builder = ensureBuildxBuilder(dockerEnv); const buildArgs = [ - "build", "--progress=plain", "--file", artifact.dockerfile, + "buildx", "build", "--builder", builder, "--load", "--progress=plain", "--file", artifact.dockerfile, "--label", `org.opencontainers.image.source=${process.env.GITHUB_SERVER_URL ?? process.env.GITEA_SERVER_URL}/${process.env.GITHUB_REPOSITORY ?? process.env.GITEA_REPOSITORY}`, "--label", `org.opencontainers.image.revision=${sha}`, "--tag", taggedImage, @@ -611,18 +659,16 @@ function buildArtifact({ artifact, registry, sha, workspace, dockerEnv, validati buildArgs.push("--build-arg", `${key}=${expandTokens(value, { sha })}`); } buildArgs.push(...resolveBuildSecretArguments(artifact.buildSecrets, dockerEnv)); - if (!validationOnly) { - try { - run("docker", ["pull", cacheImage], { cwd: workspace, env: dockerEnv, capture: true }); - buildArgs.push("--cache-from", cacheImage); - console.log(`Using registry build cache ${cacheImage}.`); - } catch { - console.log(`No reusable registry build cache is available for ${artifact.name}.`); + if (useRegistryCache) { + buildArgs.push("--cache-from", `type=registry,ref=${cacheImage}`); + if (!validationOnly) { + buildArgs.push( + "--cache-to", `type=registry,ref=${cacheImage},mode=max,ignore-error=true`, + ); + console.log(`Using read-write max-mode registry build cache ${cacheImage}.`); + } else { + console.log(`Using read-only registry build cache ${cacheImage}.`); } - buildArgs.push( - "--build-arg", "BUILDKIT_INLINE_CACHE=1", - "--tag", cacheImage, - ); } buildArgs.push(artifact.context); console.log(`Building ${artifact.name} from ${artifact.dockerfile} as ${taggedImage}.`); @@ -632,12 +678,6 @@ function buildArtifact({ artifact, registry, sha, workspace, dockerEnv, validati const pushOutput = run("docker", ["push", taggedImage], { cwd: workspace, env: dockerEnv, capture: true }); process.stdout.write(pushOutput); const digest = exactDigestFromPush(pushOutput, taggedImage); - try { - const cacheOutput = run("docker", ["push", cacheImage], { cwd: workspace, env: dockerEnv, capture: true }); - process.stdout.write(cacheOutput); - } catch (error) { - console.log(`Registry cache publication failed for ${artifact.name}; the immutable image remains valid: ${error instanceof Error ? error.message : error}`); - } return { artifact, taggedImage, digest, exactImage: `${registry}/${artifact.image}@${digest}` }; } @@ -694,29 +734,52 @@ function promoteArtifactAliases(artifacts, releaseTag, options) { } async function deployApplications({ applications, artifacts, sha, client }) { - for (const application of applications) { - const artifact = artifacts.get(application.artifact); - const environment = { - ...Object.fromEntries( - Object.entries(application.environment ?? {}).map(([key, value]) => [key, expandTokens(value, { sha })]), - ), - ...resolveSecretEnvironment(application.secretEnvironment), - }; - const result = await deployExactImage({ - client, - appId: application.appId, - image: artifact.exactImage, - registryUsername: process.env.REGISTRY_USERNAME, - registryToken: process.env.REGISTRY_TOKEN, - environment, - healthCheckTcpPort: application.healthCheckTcpPort, - postflight: application.postflight, - deploymentAttempts: application.deploymentAttempts, - deploymentRetrySeconds: application.deploymentRetrySeconds, - sha, - }); - console.log(`Deployed ${application.name}: ${result.image}`); - appendSummary(`- ${application.name}: \`${result.image}\` (${result.status})`); + const pending = new Map(applications.map((application) => [application.name, application])); + const completed = new Set(); + + while (pending.size > 0) { + const wave = applications.filter( + (application) => + pending.has(application.name) && + application.dependsOn.every((dependency) => completed.has(dependency)), + ); + if (wave.length === 0) { + throw new Error( + `Unable to resolve deployment dependencies for: ${[...pending.keys()].join(", ")}`, + ); + } + + console.log(`Deploying application wave: ${wave.map((application) => application.name).join(", ")}.`); + await Promise.all( + wave.map(async (application) => { + const artifact = artifacts.get(application.artifact); + const environment = { + ...Object.fromEntries( + Object.entries(application.environment ?? {}).map(([key, value]) => [key, expandTokens(value, { sha })]), + ), + ...resolveSecretEnvironment(application.secretEnvironment), + }; + const result = await deployExactImage({ + client, + appId: application.appId, + image: artifact.exactImage, + registryUsername: process.env.REGISTRY_USERNAME, + registryToken: process.env.REGISTRY_TOKEN, + environment, + healthCheckTcpPort: application.healthCheckTcpPort, + postflight: application.postflight, + deploymentAttempts: application.deploymentAttempts, + deploymentRetrySeconds: application.deploymentRetrySeconds, + sha, + }); + console.log(`Deployed ${application.name}: ${result.image}`); + appendSummary(`- ${application.name}: \`${result.image}\` (${result.status})`); + }), + ); + for (const application of wave) { + pending.delete(application.name); + completed.add(application.name); + } } } @@ -742,12 +805,13 @@ async function executeVersion2({ const dockerConfig = fs.mkdtempSync(path.join(os.tmpdir(), "quickstack-docker-")); const dockerEnv = { ...process.env, DOCKER_CONFIG: dockerConfig, DOCKER_BUILDKIT: "1" }; const execution = classifyVersion2Execution(pipeline, eventName); + const useRegistryCache = + execution !== "build-validation" || + isSameRepositoryPullRequest({ eventName, environment: process.env }); try { - const needsRegistryLogin = actionMode === "all" - ? execution !== "build-validation" - : actionMode === "build" - ? execution === "build-deploy" - : execution !== "build-validation"; + const needsRegistryLogin = + !(actionMode === "coordinate" && execution === "build-validation") && + (execution !== "build-validation" || useRegistryCache); if (needsRegistryLogin) dockerLogin(registry, dockerEnv); if (actionMode === "build") { @@ -767,6 +831,7 @@ async function executeVersion2({ workspace, dockerEnv, validationOnly: execution === "build-validation", + useRegistryCache, }); appendSummary( execution === "build-validation" @@ -810,6 +875,7 @@ async function executeVersion2({ workspace, dockerEnv, validationOnly: execution === "build-validation", + useRegistryCache, })); } if (execution === "build-validation") {