feat(ci): split artifact builds from deployment
This commit is contained in:
@@ -5,6 +5,14 @@ inputs:
|
|||||||
description: Path to the repository deployment manifest.
|
description: Path to the repository deployment manifest.
|
||||||
required: false
|
required: false
|
||||||
default: .quickstack/deploy.json
|
default: .quickstack/deploy.json
|
||||||
|
mode:
|
||||||
|
description: Execute the complete action, one artifact build slot or only the deployment coordinator.
|
||||||
|
required: false
|
||||||
|
default: all
|
||||||
|
artifact-index:
|
||||||
|
description: Zero-based artifact index used when mode is build.
|
||||||
|
required: false
|
||||||
|
default: "0"
|
||||||
runs:
|
runs:
|
||||||
using: composite
|
using: composite
|
||||||
steps:
|
steps:
|
||||||
@@ -12,4 +20,6 @@ runs:
|
|||||||
shell: bash
|
shell: bash
|
||||||
env:
|
env:
|
||||||
QUICKSTACK_DEPLOY_CONFIG: ${{ inputs.config-path }}
|
QUICKSTACK_DEPLOY_CONFIG: ${{ inputs.config-path }}
|
||||||
|
QUICKSTACK_ACTION_MODE: ${{ inputs.mode }}
|
||||||
|
QUICKSTACK_ARTIFACT_INDEX: ${{ inputs.artifact-index }}
|
||||||
run: node "$GITHUB_ACTION_PATH/deploy.mjs"
|
run: node "$GITHUB_ACTION_PATH/deploy.mjs"
|
||||||
|
|||||||
@@ -548,6 +548,22 @@ export function classifyVersion2Execution(pipeline, eventName) {
|
|||||||
return pipeline.strategy === "candidate" ? "build-deploy" : "promote";
|
return pipeline.strategy === "candidate" ? "build-deploy" : "promote";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function resolveActionMode(value = "all") {
|
||||||
|
const mode = String(value || "all").trim().toLowerCase();
|
||||||
|
if (!new Set(["all", "build", "coordinate"]).has(mode)) {
|
||||||
|
throw new Error(`Unsupported QuickStack action mode ${mode}.`);
|
||||||
|
}
|
||||||
|
return mode;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveArtifactIndex(value = "0") {
|
||||||
|
const index = Number(value);
|
||||||
|
if (!Number.isInteger(index) || index < 0) {
|
||||||
|
throw new Error("QuickStack artifact index must be a non-negative integer.");
|
||||||
|
}
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
|
||||||
export function validatePromotionPullRequestSource(pipeline, headBranch) {
|
export function validatePromotionPullRequestSource(pipeline, headBranch) {
|
||||||
const sourceBranch = requiredString(pipeline.source?.branch, "Promotion source branch");
|
const sourceBranch = requiredString(pipeline.source?.branch, "Promotion source branch");
|
||||||
const actualHeadBranch = requiredString(headBranch, "Pull request head branch");
|
const actualHeadBranch = requiredString(headBranch, "Pull request head branch");
|
||||||
@@ -584,6 +600,7 @@ function verifyRequiredContainerFiles(taggedImage, requiredFiles, options) {
|
|||||||
|
|
||||||
function buildArtifact({ artifact, registry, sha, workspace, dockerEnv, validationOnly }) {
|
function buildArtifact({ artifact, registry, sha, workspace, dockerEnv, validationOnly }) {
|
||||||
const taggedImage = `${registry}/${artifact.image}:sha-${sha}`;
|
const taggedImage = `${registry}/${artifact.image}:sha-${sha}`;
|
||||||
|
const cacheImage = `${registry}/${artifact.image}:buildcache`;
|
||||||
const buildArgs = [
|
const buildArgs = [
|
||||||
"build", "--progress=plain", "--file", artifact.dockerfile,
|
"build", "--progress=plain", "--file", artifact.dockerfile,
|
||||||
"--label", `org.opencontainers.image.source=${process.env.GITHUB_SERVER_URL ?? process.env.GITEA_SERVER_URL}/${process.env.GITHUB_REPOSITORY ?? process.env.GITEA_REPOSITORY}`,
|
"--label", `org.opencontainers.image.source=${process.env.GITHUB_SERVER_URL ?? process.env.GITEA_SERVER_URL}/${process.env.GITHUB_REPOSITORY ?? process.env.GITEA_REPOSITORY}`,
|
||||||
@@ -594,6 +611,19 @@ function buildArtifact({ artifact, registry, sha, workspace, dockerEnv, validati
|
|||||||
buildArgs.push("--build-arg", `${key}=${expandTokens(value, { sha })}`);
|
buildArgs.push("--build-arg", `${key}=${expandTokens(value, { sha })}`);
|
||||||
}
|
}
|
||||||
buildArgs.push(...resolveBuildSecretArguments(artifact.buildSecrets, dockerEnv));
|
buildArgs.push(...resolveBuildSecretArguments(artifact.buildSecrets, dockerEnv));
|
||||||
|
if (!validationOnly) {
|
||||||
|
try {
|
||||||
|
run("docker", ["pull", cacheImage], { cwd: workspace, env: dockerEnv, capture: true });
|
||||||
|
buildArgs.push("--cache-from", cacheImage);
|
||||||
|
console.log(`Using registry build cache ${cacheImage}.`);
|
||||||
|
} catch {
|
||||||
|
console.log(`No reusable registry build cache is available for ${artifact.name}.`);
|
||||||
|
}
|
||||||
|
buildArgs.push(
|
||||||
|
"--build-arg", "BUILDKIT_INLINE_CACHE=1",
|
||||||
|
"--tag", cacheImage,
|
||||||
|
);
|
||||||
|
}
|
||||||
buildArgs.push(artifact.context);
|
buildArgs.push(artifact.context);
|
||||||
console.log(`Building ${artifact.name} from ${artifact.dockerfile} as ${taggedImage}.`);
|
console.log(`Building ${artifact.name} from ${artifact.dockerfile} as ${taggedImage}.`);
|
||||||
run("docker", buildArgs, { cwd: workspace, env: dockerEnv });
|
run("docker", buildArgs, { cwd: workspace, env: dockerEnv });
|
||||||
@@ -602,6 +632,12 @@ function buildArtifact({ artifact, registry, sha, workspace, dockerEnv, validati
|
|||||||
const pushOutput = run("docker", ["push", taggedImage], { cwd: workspace, env: dockerEnv, capture: true });
|
const pushOutput = run("docker", ["push", taggedImage], { cwd: workspace, env: dockerEnv, capture: true });
|
||||||
process.stdout.write(pushOutput);
|
process.stdout.write(pushOutput);
|
||||||
const digest = exactDigestFromPush(pushOutput, taggedImage);
|
const digest = exactDigestFromPush(pushOutput, taggedImage);
|
||||||
|
try {
|
||||||
|
const cacheOutput = run("docker", ["push", cacheImage], { cwd: workspace, env: dockerEnv, capture: true });
|
||||||
|
process.stdout.write(cacheOutput);
|
||||||
|
} catch (error) {
|
||||||
|
console.log(`Registry cache publication failed for ${artifact.name}; the immutable image remains valid: ${error instanceof Error ? error.message : error}`);
|
||||||
|
}
|
||||||
return { artifact, taggedImage, digest, exactImage: `${registry}/${artifact.image}@${digest}` };
|
return { artifact, taggedImage, digest, exactImage: `${registry}/${artifact.image}@${digest}` };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -684,7 +720,15 @@ async function deployApplications({ applications, artifacts, sha, client }) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function executeVersion2({ config, pipeline, eventName, sha, workspace }) {
|
async function executeVersion2({
|
||||||
|
config,
|
||||||
|
pipeline,
|
||||||
|
eventName,
|
||||||
|
sha,
|
||||||
|
workspace,
|
||||||
|
actionMode = "all",
|
||||||
|
artifactIndex = 0,
|
||||||
|
}) {
|
||||||
const commitMessage = run("git", ["log", "-1", "--pretty=%B"], { cwd: workspace, capture: true });
|
const commitMessage = run("git", ["log", "-1", "--pretty=%B"], { cwd: workspace, capture: true });
|
||||||
const validationOnly = eventName === "pull_request";
|
const validationOnly = eventName === "pull_request";
|
||||||
const skipMarker = pipeline.strategy === "promote"
|
const skipMarker = pipeline.strategy === "promote"
|
||||||
@@ -696,10 +740,66 @@ async function executeVersion2({ config, pipeline, eventName, sha, workspace })
|
|||||||
}
|
}
|
||||||
const registry = requiredString(config.registry ?? "gitea.nuvisphere.de", "OCI registry").replace(/\/$/, "");
|
const registry = requiredString(config.registry ?? "gitea.nuvisphere.de", "OCI registry").replace(/\/$/, "");
|
||||||
const dockerConfig = fs.mkdtempSync(path.join(os.tmpdir(), "quickstack-docker-"));
|
const dockerConfig = fs.mkdtempSync(path.join(os.tmpdir(), "quickstack-docker-"));
|
||||||
const dockerEnv = { ...process.env, DOCKER_CONFIG: dockerConfig };
|
const dockerEnv = { ...process.env, DOCKER_CONFIG: dockerConfig, DOCKER_BUILDKIT: "1" };
|
||||||
const execution = classifyVersion2Execution(pipeline, eventName);
|
const execution = classifyVersion2Execution(pipeline, eventName);
|
||||||
try {
|
try {
|
||||||
if (execution !== "build-validation") dockerLogin(registry, dockerEnv);
|
const needsRegistryLogin = actionMode === "all"
|
||||||
|
? execution !== "build-validation"
|
||||||
|
: actionMode === "build"
|
||||||
|
? execution === "build-deploy"
|
||||||
|
: execution !== "build-validation";
|
||||||
|
if (needsRegistryLogin) dockerLogin(registry, dockerEnv);
|
||||||
|
|
||||||
|
if (actionMode === "build") {
|
||||||
|
if (execution !== "build-validation" && execution !== "build-deploy") {
|
||||||
|
console.log(`Artifact build slot ${artifactIndex} is not needed for ${execution}.`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const artifact = pipeline.artifacts[artifactIndex];
|
||||||
|
if (!artifact) {
|
||||||
|
console.log(`Artifact build slot ${artifactIndex} is unused for ${pipeline.name ?? pipeline.branch}.`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const built = buildArtifact({
|
||||||
|
artifact,
|
||||||
|
registry,
|
||||||
|
sha,
|
||||||
|
workspace,
|
||||||
|
dockerEnv,
|
||||||
|
validationOnly: execution === "build-validation",
|
||||||
|
});
|
||||||
|
appendSummary(
|
||||||
|
execution === "build-validation"
|
||||||
|
? `Validated OCI artifact \`${artifact.name}\` for \`${pipeline.branch}\`.`
|
||||||
|
: `Built OCI artifact \`${artifact.name}\`: \`${built.exactImage}\`.`,
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (actionMode === "coordinate" && execution === "build-validation") {
|
||||||
|
appendSummary(`Validated ${pipeline.artifacts.length} immutable OCI artifact build(s) for \`${pipeline.branch}\`.`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (actionMode === "coordinate" && execution === "build-deploy") {
|
||||||
|
const artifacts = new Map();
|
||||||
|
for (const artifact of pipeline.artifacts) {
|
||||||
|
artifacts.set(artifact.name, pullCandidateArtifact({
|
||||||
|
artifact,
|
||||||
|
registry,
|
||||||
|
sourceSha: sha,
|
||||||
|
workspace,
|
||||||
|
dockerEnv,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
const client = createQuickStackClient({
|
||||||
|
baseUrl: process.env.QUICKSTACK_BASE_URL,
|
||||||
|
token: process.env.QUICKSTACK_API_TOKEN,
|
||||||
|
});
|
||||||
|
await deployApplications({ applications: pipeline.applications, artifacts, sha, client });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (execution === "build-validation" || execution === "build-deploy") {
|
if (execution === "build-validation" || execution === "build-deploy") {
|
||||||
const artifacts = new Map();
|
const artifacts = new Map();
|
||||||
for (const artifact of pipeline.artifacts) {
|
for (const artifact of pipeline.artifacts) {
|
||||||
@@ -789,6 +889,8 @@ async function main() {
|
|||||||
const config = JSON.parse(fs.readFileSync(configPath, "utf8"));
|
const config = JSON.parse(fs.readFileSync(configPath, "utf8"));
|
||||||
const eventName = process.env.GITHUB_EVENT_NAME ?? process.env.GITEA_EVENT_NAME ?? "";
|
const eventName = process.env.GITHUB_EVENT_NAME ?? process.env.GITEA_EVENT_NAME ?? "";
|
||||||
const branch = resolveDeploymentBranch({ eventName });
|
const branch = resolveDeploymentBranch({ eventName });
|
||||||
|
const actionMode = resolveActionMode(process.env.QUICKSTACK_ACTION_MODE);
|
||||||
|
const artifactIndex = resolveArtifactIndex(process.env.QUICKSTACK_ARTIFACT_INDEX);
|
||||||
if (config.version === 2) {
|
if (config.version === 2) {
|
||||||
const pipeline = validateVersion2Pipeline(config, branch);
|
const pipeline = validateVersion2Pipeline(config, branch);
|
||||||
if (!pipeline) {
|
if (!pipeline) {
|
||||||
@@ -797,7 +899,19 @@ async function main() {
|
|||||||
}
|
}
|
||||||
const sha = requiredString(process.env.GITHUB_SHA ?? process.env.GITEA_SHA, "Git SHA").toLowerCase();
|
const sha = requiredString(process.env.GITHUB_SHA ?? process.env.GITEA_SHA, "Git SHA").toLowerCase();
|
||||||
if (!/^[0-9a-f]{40,64}$/.test(sha)) throw new Error("Git SHA must be a full hexadecimal commit ID.");
|
if (!/^[0-9a-f]{40,64}$/.test(sha)) throw new Error("Git SHA must be a full hexadecimal commit ID.");
|
||||||
await executeVersion2({ config, pipeline, eventName, sha, workspace });
|
await executeVersion2({
|
||||||
|
config,
|
||||||
|
pipeline,
|
||||||
|
eventName,
|
||||||
|
sha,
|
||||||
|
workspace,
|
||||||
|
actionMode,
|
||||||
|
artifactIndex,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (actionMode === "build") {
|
||||||
|
console.log("Manifest version 1 remains on the coordinator's serial compatibility path.");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const deployment = selectDeployment(config, branch);
|
const deployment = selectDeployment(config, branch);
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import {
|
|||||||
deployExactImage,
|
deployExactImage,
|
||||||
expandTokens,
|
expandTokens,
|
||||||
mergeEnvironment,
|
mergeEnvironment,
|
||||||
|
resolveActionMode,
|
||||||
|
resolveArtifactIndex,
|
||||||
resolveBuildSecretArguments,
|
resolveBuildSecretArguments,
|
||||||
resolveSecretEnvironment,
|
resolveSecretEnvironment,
|
||||||
orderApplications,
|
orderApplications,
|
||||||
@@ -100,6 +102,16 @@ test("production pull requests validate tested candidates without rebuilding", (
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("validates split action modes and artifact build slots", () => {
|
||||||
|
assert.equal(resolveActionMode(undefined), "all");
|
||||||
|
assert.equal(resolveActionMode("BUILD"), "build");
|
||||||
|
assert.equal(resolveActionMode("coordinate"), "coordinate");
|
||||||
|
assert.throws(() => resolveActionMode("parallel"), /Unsupported QuickStack action mode/);
|
||||||
|
assert.equal(resolveArtifactIndex("2"), 2);
|
||||||
|
assert.throws(() => resolveArtifactIndex("-1"), /non-negative integer/);
|
||||||
|
assert.throws(() => resolveArtifactIndex("1.5"), /non-negative integer/);
|
||||||
|
});
|
||||||
|
|
||||||
test("validates OCI paths and repository-local build paths", () => {
|
test("validates OCI paths and repository-local build paths", () => {
|
||||||
assert.equal(validateTarget({ name: "Web", image: "Owner/Web", appId: "app-1" }).image, "owner/web");
|
assert.equal(validateTarget({ name: "Web", image: "Owner/Web", appId: "app-1" }).image, "owner/web");
|
||||||
assert.throws(() => validateTarget({ name: "Web", image: "owner/web", appId: "app-1", context: "../secret" }), /inside/);
|
assert.throws(() => validateTarget({ name: "Web", image: "owner/web", appId: "app-1", context: "../secret" }), /inside/);
|
||||||
|
|||||||
@@ -97,3 +97,11 @@ the command line as build arguments. The Dockerfile consumes them with
|
|||||||
`RUN --mount=type=secret,id=<id>,required=true ...`.
|
`RUN --mount=type=secret,id=<id>,required=true ...`.
|
||||||
|
|
||||||
Candidate pipelines build every artifact once, push `sha-<commit>`, resolve the registry digest and deploy applications in dependency order. A pull request into a promotion branch must originate from `source.branch`; it pulls and verifies the already tested `sha-<commit>` candidates without rebuilding or deploying them. Promotion pushes require a merge parent with an identical Git tree, pull the existing candidate, verify required container files, add the SemVer alias, deploy the exact digests, create the immutable tag and publish the canonical Gitea release. Tag creation does not trigger another scoped pipeline because the workflow listens only to branch pushes.
|
Candidate pipelines build every artifact once, push `sha-<commit>`, resolve the registry digest and deploy applications in dependency order. A pull request into a promotion branch must originate from `source.branch`; it pulls and verifies the already tested `sha-<commit>` candidates without rebuilding or deploying them. Promotion pushes require a merge parent with an identical Git tree, pull the existing candidate, verify required container files, add the SemVer alias, deploy the exact digests, create the immutable tag and publish the canonical Gitea release. Tag creation does not trigger another scoped pipeline because the workflow listens only to branch pushes.
|
||||||
|
|
||||||
|
Scoped workflows can distribute version 2 artifact builds across independent
|
||||||
|
runner jobs by invoking the action with `mode: build` and a zero-based
|
||||||
|
`artifact-index`. A final job invokes `mode: coordinate` after all build jobs.
|
||||||
|
Candidate tags are the synchronization boundary, so exact digests never depend
|
||||||
|
on matrix output merging. Published candidates also maintain a per-image
|
||||||
|
`buildcache` tag with inline BuildKit metadata for ephemeral runners. Version 1
|
||||||
|
manifests remain on the serial coordinator compatibility path.
|
||||||
|
|||||||
Reference in New Issue
Block a user