diff --git a/src/spec-common/cliHost.ts b/src/spec-common/cliHost.ts index 294f8be4a..85d0d47fd 100644 --- a/src/spec-common/cliHost.ts +++ b/src/spec-common/cliHost.ts @@ -7,7 +7,7 @@ import * as path from 'path'; import * as net from 'net'; import * as os from 'os'; -import { readLocalFile, writeLocalFile, mkdirpLocal, isLocalFile, renameLocal, readLocalDir, isLocalFolder } from '../spec-utils/pfs'; +import { readLocalFile, writeLocalFile, mkdirpLocal, isLocalFile, renameLocal, readLocalDir, isLocalFolder, unlinkLocal } from '../spec-utils/pfs'; import { URI } from 'vscode-uri'; import { ExecFunction, getLocalUsername, plainExec, plainPtyExec, PtyExecFunction } from './commonUtils'; import { Abort, Duplex, Sink, Source, SourceCallback } from 'pull-stream'; @@ -32,6 +32,7 @@ export interface CLIHost { isFolder(filepath: string): Promise; readFile(filepath: string): Promise; writeFile(filepath: string, content: Buffer): Promise; + deleteFile?(filepath: string): Promise; rename(oldPath: string, newPath: string): Promise; mkdirp(dirpath: string): Promise; readDir(dirpath: string): Promise; @@ -76,6 +77,7 @@ function createLocalCLIHostFromExecFunctions(localCwd: string, exec: ExecFunctio isFolder: isLocalFolder, readFile: readLocalFile, writeFile: writeLocalFile, + deleteFile: unlinkLocal, rename: renameLocal, mkdirp: async (dirpath) => { await mkdirpLocal(dirpath); diff --git a/src/spec-node/dockerCompose.ts b/src/spec-node/dockerCompose.ts index 8093464cc..6ceef7231 100644 --- a/src/spec-node/dockerCompose.ts +++ b/src/spec-node/dockerCompose.ts @@ -19,6 +19,7 @@ import { Mount, parseMount } from '../spec-configuration/containerFeaturesConfig import path from 'path'; import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageBuildInfoFromImage, getImageMetadataFromContainer, ImageBuildInfo, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata'; import { ensureDockerfileHasFinalStageName } from './dockerfileUtils'; +import { materializeResolvedDockerfileForBuild, resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess'; import { randomUUID } from 'crypto'; const projectLabel = 'com.docker.compose.project'; @@ -163,11 +164,16 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf let baseName = 'dev_container_auto_added_stage_label'; let dockerfile: string | undefined; let imageBuildInfo: ImageBuildInfo; + let preprocessedDockerfilePathForComposeBuild: string | undefined; + let disposeMaterializedDockerfile = async () => { }; + let resolvedBuildDockerfile; const serviceInfo = getBuildInfoForService(composeService, cliHost.path, localComposeFiles); if (serviceInfo.build) { const { context, dockerfilePath, target } = serviceInfo.build; const resolvedDockerfilePath = cliHost.path.isAbsolute(dockerfilePath) ? dockerfilePath : path.resolve(context, dockerfilePath); - const originalDockerfile = (await cliHost.readFile(resolvedDockerfilePath)).toString(); + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, resolvedDockerfilePath); + resolvedBuildDockerfile = resolvedDockerfile; + const originalDockerfile = resolvedDockerfile.effectiveDockerfileContent; dockerfile = originalDockerfile; if (target) { // Explictly set build target for the dev container build features on that @@ -191,9 +197,18 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf const supportsAdditionalBuildContexts = !params.isPodman && version && !isEarlierVersion(version, [2, 17, 0]); const optionalBuildKitParams = supportsAdditionalBuildContexts ? params : { ...params, buildKitVersion: undefined }; const extendImageBuildInfo = await getExtendImageBuildInfo(optionalBuildKitParams, configWithRaw, baseName, imageBuildInfo, composeService.user, additionalFeatures, canAddLabelsToContainer); + if (resolvedBuildDockerfile && !extendImageBuildInfo?.featureBuildInfo) { + const materializedDockerfile = await materializeResolvedDockerfileForBuild(cliHost, resolvedBuildDockerfile); + preprocessedDockerfilePathForComposeBuild = materializedDockerfile.dockerfilePath; + disposeMaterializedDockerfile = materializedDockerfile.dispose; + } let overrideImageName: string | undefined; let buildOverrideContent = ''; + if (preprocessedDockerfilePathForComposeBuild && !extendImageBuildInfo?.featureBuildInfo) { + buildOverrideContent += ' build:\n'; + buildOverrideContent += ` dockerfile: ${preprocessedDockerfilePathForComposeBuild}\n`; + } if (extendImageBuildInfo?.featureBuildInfo) { // Avoid retagging a previously pulled image. if (!serviceInfo.build) { @@ -264,7 +279,8 @@ ${cacheFromOverrideContent} args.push('-f', composeOverrideFile); } - if (!noBuild) { + try { + if (!noBuild) { args.push('build'); if (noCache) { args.push('--no-cache'); @@ -294,6 +310,9 @@ ${cacheFromOverrideContent} throw err instanceof ContainerError ? err : new ContainerError({ description: 'An error occurred building the Docker Compose images.', originalError: err, data: { fileWithError: localComposeFiles[0] } }); } + } + } finally { + await disposeMaterializedDockerfile(); } return { diff --git a/src/spec-node/dockerfilePreprocess.ts b/src/spec-node/dockerfilePreprocess.ts new file mode 100644 index 000000000..c2dc5ac84 --- /dev/null +++ b/src/spec-node/dockerfilePreprocess.ts @@ -0,0 +1,395 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CLIHost } from '../spec-common/cliHost'; +import { ContainerError } from '../spec-common/errors'; +import { randomUUID } from 'crypto'; + +const preprocessorDirective = /^\s*#\s*(\w+)\b(.*)$/; +const includeLine = /^\s*"([^"]+)"\s*$/; +const defineLine = /^\s*([a-zA-Z_][a-zA-Z0-9_]*)(?:\s+(.*))?$/; +const undefLine = /^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*$/; +const ifdefLine = /^\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*$/; +const fromLine = /^\s*FROM(?:\s|$)/mi; + +interface ConditionalState { + parentActive: boolean; + thisActive: boolean; + hasMatched: boolean; +} + +export interface ResolvedDockerfile { + originalDockerfilePath: string; + effectiveDockerfileContent: string; + preprocessed: boolean; +} + +export interface MaterializedDockerfile { + dockerfilePath: string; + dispose(): Promise; +} + + +/** + * Preprocesses a Dockerfile, simulating cpp -E style preprocessing for Podman compatibility. + * If the file ends with .in, it will resolve #include statements recursively and return the rewritten Dockerfile content. + * Otherwise, returns the original Dockerfile content. + * + * @param cliHost CLIHost for file operations + * @param dockerfilePath Path to the Dockerfile (may be .in) + * @returns { originalDockerfilePath, effectiveDockerfileContent, preprocessed } + */ +export async function resolveDockerfileIncludesIfNeeded(cliHost: CLIHost, dockerfilePath: string): Promise { + const dockerfileText = (await cliHost.readFile(dockerfilePath)).toString(); + if (!dockerfilePath.toLowerCase().endsWith('.in')) { + return { + originalDockerfilePath: dockerfilePath, + effectiveDockerfileContent: dockerfileText, + preprocessed: false, + }; + } + + const rewrittenContent = await preprocessDockerfileIncludes(cliHost, dockerfilePath, [], new Map()); + validateResolvedFromInstruction(dockerfilePath, rewrittenContent); + + return { + originalDockerfilePath: dockerfilePath, + effectiveDockerfileContent: rewrittenContent, + preprocessed: true, + }; +} + +export async function materializeResolvedDockerfileForBuild(cliHost: CLIHost, resolvedDockerfile: ResolvedDockerfile): Promise { + if (!resolvedDockerfile.preprocessed) { + return { + dockerfilePath: resolvedDockerfile.originalDockerfilePath, + dispose: async () => { }, + }; + } + + const dockerfilePath = await writePreprocessedDockerfile(cliHost, resolvedDockerfile.originalDockerfilePath, resolvedDockerfile.effectiveDockerfileContent); + return { + dockerfilePath, + dispose: async () => { + if (cliHost.deleteFile) { + await cliHost.deleteFile(dockerfilePath); + } + }, + }; +} + +async function preprocessDockerfileIncludes(cliHost: CLIHost, currentPath: string, stack: string[], macros: Map): Promise { + if (stack.includes(currentPath)) { + const chain = [...stack, currentPath].join(' -> '); + throw new ContainerError({ description: `Cyclic #include detected while preprocessing Dockerfile: ${chain}` }); + } + if (!(await cliHost.isFile(currentPath))) { + throw new ContainerError({ description: `Included Dockerfile not found: ${currentPath}` }); + } + + const currentText = (await cliHost.readFile(currentPath)).toString(); + const lines = currentText.split(/\r?\n/); + const expanded: string[] = []; + const nextStack = [...stack, currentPath]; + const conditionals: ConditionalState[] = []; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const lineNumber = i + 1; + const currentActive = conditionals.length === 0 ? true : conditionals[conditionals.length - 1].thisActive; + const directive = preprocessorDirective.exec(line); + + if (!directive) { + if (currentActive) { + expanded.push(substituteMacros(line, macros)); + } + continue; + } + + const directiveName = directive[1].toLowerCase(); + const directiveBody = directive[2] ?? ''; + + if (directiveName === 'if' || directiveName === 'ifdef' || directiveName === 'ifndef') { + const parentActive = currentActive; + const condition = evaluateIfDirectiveCondition(currentPath, lineNumber, directiveName, directiveBody, macros); + const thisActive = parentActive && condition; + conditionals.push({ + parentActive, + thisActive, + hasMatched: thisActive, + }); + continue; + } + + if (directiveName === 'elif') { + const state = conditionals[conditionals.length - 1]; + if (!state) { + throw new ContainerError({ description: `#elif without matching #if in ${currentPath}:${lineNumber}` }); + } + if (!state.parentActive || state.hasMatched) { + state.thisActive = false; + continue; + } + const elifCondition = evaluateIfDirectiveCondition(currentPath, lineNumber, directiveName, directiveBody, macros); + state.thisActive = state.parentActive && elifCondition; + if (state.thisActive) { + state.hasMatched = true; + } + continue; + } + + if (directiveName === 'else') { + const state = conditionals[conditionals.length - 1]; + if (!state) { + throw new ContainerError({ description: `#else without matching #if in ${currentPath}:${lineNumber}` }); + } + state.thisActive = state.parentActive && !state.hasMatched; + state.hasMatched = true; + continue; + } + + if (directiveName === 'endif') { + const state = conditionals.pop(); + if (!state) { + throw new ContainerError({ description: `#endif without matching #if in ${currentPath}:${lineNumber}` }); + } + continue; + } + + if (!currentActive) { + continue; + } + + switch (directiveName) { + case 'include': { + const includeMatch = includeLine.exec(directiveBody); + if (!includeMatch) { + throw new ContainerError({ description: `Invalid #include directive in ${currentPath}:${lineNumber}. Use #include "path".` }); + } + const includePath = substituteMacros(includeMatch[1], macros); + const resolvedIncludePath = cliHost.path.isAbsolute(includePath) + ? includePath + : cliHost.path.resolve(cliHost.path.dirname(currentPath), includePath); + expanded.push(await preprocessDockerfileIncludes(cliHost, resolvedIncludePath, nextStack, macros)); + break; + } + case 'define': { + const defineMatch = defineLine.exec(directiveBody); + if (!defineMatch) { + throw new ContainerError({ description: `Invalid #define directive in ${currentPath}:${lineNumber}.` }); + } + macros.set(defineMatch[1], defineMatch[2] ?? '1'); + break; + } + case 'undef': { + const undefMatch = undefLine.exec(directiveBody); + if (!undefMatch) { + throw new ContainerError({ description: `Invalid #undef directive in ${currentPath}:${lineNumber}.` }); + } + macros.delete(undefMatch[1]); + break; + } + case 'error': { + const message = substituteMacros(directiveBody.trim(), macros); + throw new ContainerError({ description: `#error in ${currentPath}:${lineNumber}: ${message}` }); + } + case 'warning': { + const message = substituteMacros(directiveBody.trim(), macros); + expanded.push(`# warning: ${message}`); + break; + } + default: + expanded.push(substituteMacros(line, macros)); + break; + } + } + + if (conditionals.length > 0) { + throw new ContainerError({ description: `Unterminated preprocessor conditionals in ${currentPath}. Missing #endif.` }); + } + + return expanded.join('\n'); +} + +function evaluateIfDirectiveCondition(currentPath: string, lineNumber: number, directiveName: string, body: string, macros: Map): boolean { + if (directiveName === 'ifdef' || directiveName === 'ifndef') { + const ifdefMatch = ifdefLine.exec(body); + if (!ifdefMatch) { + throw new ContainerError({ description: `Invalid #${directiveName} directive in ${currentPath}:${lineNumber}.` }); + } + const isDefined = macros.has(ifdefMatch[1]); + return directiveName === 'ifdef' ? isDefined : !isDefined; + } + + return evaluateBooleanExpression(body.trim(), macros); +} + +function evaluateBooleanExpression(expression: string, macros: Map): boolean { + if (!expression) { + return false; + } + + const tokens = tokenizeExpression(expression); + let index = 0; + + const parseExpression = (): boolean => { + let value = parseTerm(); + while (tokens[index] === '||') { + index++; + value = value || parseTerm(); + } + return value; + }; + + const parseTerm = (): boolean => { + let value = parseFactor(); + while (tokens[index] === '&&') { + index++; + value = value && parseFactor(); + } + return value; + }; + + const parseFactor = (): boolean => { + const token = tokens[index]; + if (token === '!') { + index++; + return !parseFactor(); + } + if (token === '(') { + index++; + const value = parseExpression(); + if (tokens[index] !== ')') { + throw new ContainerError({ description: `Invalid #if expression: missing ')' in '${expression}'` }); + } + index++; + return value; + } + if (token === 'defined') { + index++; + if (tokens[index] === '(') { + index++; + const name = tokens[index++]; + if (!name || !isIdentifier(name)) { + throw new ContainerError({ description: `Invalid #if expression: expected identifier after defined(` }); + } + if (tokens[index] !== ')') { + throw new ContainerError({ description: `Invalid #if expression: missing ')' after defined(${name}` }); + } + index++; + return macros.has(name); + } + const name = tokens[index++]; + if (!name || !isIdentifier(name)) { + throw new ContainerError({ description: 'Invalid #if expression: expected identifier after defined' }); + } + return macros.has(name); + } + + if (!token) { + throw new ContainerError({ description: `Invalid #if expression: unexpected end of expression '${expression}'` }); + } + index++; + if (/^[+-]?\d+$/.test(token)) { + return Number(token) !== 0; + } + if (isIdentifier(token)) { + return evaluateMacroTruthiness(token, macros, new Set()); + } + throw new ContainerError({ description: `Invalid #if expression token '${token}'` }); + }; + + const result = parseExpression(); + if (index !== tokens.length) { + throw new ContainerError({ description: `Invalid #if expression near '${tokens.slice(index).join(' ')}'` }); + } + return result; +} + +function tokenizeExpression(expression: string): string[] { + const tokens = expression.match(/\s+|\|\||&&|!|\(|\)|defined|[a-zA-Z_][a-zA-Z0-9_]*|[+-]?\d+/g) || []; + let reconstructed = ''; + const filtered = tokens.filter(token => token.trim().length > 0); + for (const token of tokens) { + reconstructed += token; + } + const normalizedExpression = expression.replace(/\s+/g, ''); + if (reconstructed.replace(/\s+/g, '') !== normalizedExpression) { + throw new ContainerError({ description: `Unsupported token in #if expression '${expression}'` }); + } + return filtered; +} + +function isIdentifier(token: string): boolean { + return /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(token); +} + +function evaluateMacroTruthiness(name: string, macros: Map, seen: Set): boolean { + if (seen.has(name)) { + return false; + } + const value = macros.get(name); + if (typeof value !== 'string') { + return false; + } + const trimmed = value.trim(); + if (!trimmed || trimmed === '0' || trimmed.toLowerCase() === 'false') { + return false; + } + if (/^[+-]?\d+$/.test(trimmed)) { + return Number(trimmed) !== 0; + } + if (isIdentifier(trimmed)) { + seen.add(name); + return evaluateMacroTruthiness(trimmed, macros, seen); + } + return true; +} + +function substituteMacros(line: string, macros: Map): string { + if (!macros.size) { + return line; + } + + let result = line; + for (let i = 0; i < 10; i++) { + let changed = false; + const names = [...macros.keys()].sort((a, b) => b.length - a.length); + for (const name of names) { + const value = macros.get(name)!; + const pattern = new RegExp(`\\b${escapeRegExp(name)}\\b`, 'g'); + const replaced = result.replace(pattern, value); + if (replaced !== result) { + changed = true; + result = replaced; + } + } + if (!changed) { + break; + } + } + + return result; +} + +function escapeRegExp(str: string): string { + return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function validateResolvedFromInstruction(originalDockerfilePath: string, dockerfileContent: string) { + if (!fromLine.test(dockerfileContent)) { + throw new ContainerError({ + description: `Preprocessed Dockerfile '${originalDockerfilePath}' contains no resolved FROM instruction. Ensure preprocessing directives produce at least one final FROM line.`, + }); + } +} + +async function writePreprocessedDockerfile(cliHost: CLIHost, originalDockerfilePath: string, dockerfileContent: string): Promise { + const dockerfileFolder = cliHost.path.dirname(originalDockerfilePath); + const outputFileName = `${Date.now()}-${randomUUID()}-${cliHost.path.basename(originalDockerfilePath).replace(/\.in$/i, '')}`; + const effectiveDockerfilePath = cliHost.path.join(dockerfileFolder, outputFileName); + await cliHost.writeFile(effectiveDockerfilePath, Buffer.from(dockerfileContent)); + return effectiveDockerfilePath; +} \ No newline at end of file diff --git a/src/spec-node/imageMetadata.ts b/src/spec-node/imageMetadata.ts index 60884592e..b4839e2cc 100644 --- a/src/spec-node/imageMetadata.ts +++ b/src/spec-node/imageMetadata.ts @@ -12,6 +12,7 @@ import { ContainerDetails, DockerCLIParameters, ImageDetails } from '../spec-shu import { Log, LogLevel } from '../spec-utils/log'; import { getBuildInfoForService, readDockerComposeConfig } from './dockerCompose'; import { Dockerfile, extractDockerfile, findBaseImage, findUserStatement } from './dockerfileUtils'; +import { resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess'; import { SubstituteConfig, SubstitutedConfig, DockerResolverParameters, inspectDockerImage, uriToWSLFsPath, envListToObj } from './utils'; const pickConfigProperties: (keyof DevContainerConfig & keyof ImageMetadataEntry)[] = [ @@ -342,7 +343,8 @@ export async function getImageBuildInfo(params: DockerResolverParameters | Docke if (!cliHost.isFile(dockerfilePath)) { throw new ContainerError({ description: `Dockerfile (${dockerfilePath}) not found.` }); } - const dockerfile = (await cliHost.readFile(dockerfilePath)).toString(); + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, dockerfilePath); + const dockerfile = resolvedDockerfile.effectiveDockerfileContent; return getImageBuildInfoFromDockerfile(params, dockerfile, config.build?.args || {}, config.build?.target, configWithRaw.substitute); } else if ('dockerComposeFile' in config) { @@ -363,7 +365,8 @@ export async function getImageBuildInfo(params: DockerResolverParameters | Docke if (serviceInfo.build) { const { context, dockerfilePath } = serviceInfo.build; const resolvedDockerfilePath = cliHost.path.isAbsolute(dockerfilePath) ? dockerfilePath : cliHost.path.resolve(context, dockerfilePath); - const dockerfile = (await cliHost.readFile(resolvedDockerfilePath)).toString(); + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, resolvedDockerfilePath); + const dockerfile = resolvedDockerfile.effectiveDockerfileContent; return getImageBuildInfoFromDockerfile(params, dockerfile, serviceInfo.build.args || {}, serviceInfo.build.target, configWithRaw.substitute); } else { return getImageBuildInfoFromImage(params, composeService.image, configWithRaw.substitute); diff --git a/src/spec-node/singleContainer.ts b/src/spec-node/singleContainer.ts index 1c3669f74..d6ad4749d 100644 --- a/src/spec-node/singleContainer.ts +++ b/src/spec-node/singleContainer.ts @@ -13,6 +13,7 @@ import { LogLevel, Log, makeLog } from '../spec-utils/log'; import { extendImage, getExtendImageBuildInfo, updateRemoteUserUID } from './containerFeatures'; import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageMetadataFromContainer, ImageMetadataEntry, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata'; import { ensureDockerfileHasFinalStageName, generateMountCommand } from './dockerfileUtils'; +import { materializeResolvedDockerfileForBuild, resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess'; export const hostFolderLabel = 'devcontainer.local_folder'; // used to label containers created from a workspace/folder export const configFileLabel = 'devcontainer.config_file'; @@ -130,7 +131,8 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config throw new ContainerError({ description: `Dockerfile (${dockerfilePath}) not found.` }); } - let dockerfile = (await cliHost.readFile(dockerfilePath)).toString(); + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, dockerfilePath); + let dockerfile = resolvedDockerfile.effectiveDockerfileContent; const originalDockerfile = dockerfile; let baseName = 'dev_container_auto_added_stage_label'; if (config.build?.target) { @@ -150,6 +152,7 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config const extendImageBuildInfo = await getExtendImageBuildInfo(buildParams, configWithRaw, baseName, imageBuildInfo, undefined, additionalFeatures, false); let finalDockerfilePath = dockerfilePath; + let disposeMaterializedDockerfile = async () => { }; const additionalBuildArgs: string[] = []; if (extendImageBuildInfo?.featureBuildInfo) { const { featureBuildInfo } = extendImageBuildInfo; @@ -173,6 +176,10 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config for (const securityOpt of featureBuildInfo.securityOpts) { additionalBuildArgs.push('--security-opt', securityOpt); } + } else { + const materializedDockerfile = await materializeResolvedDockerfileForBuild(cliHost, resolvedDockerfile); + finalDockerfilePath = materializedDockerfile.dockerfilePath; + disposeMaterializedDockerfile = materializedDockerfile.dispose; } const args: string[] = []; @@ -261,6 +268,8 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config } throw new ContainerError({ description: 'An error occurred building the image.', originalError: err, data: { fileWithError: dockerfilePath } }); + } finally { + await disposeMaterializedDockerfile(); } const imageDetails = () => inspectDockerImage(buildParams, baseImageNames[0], false); diff --git a/src/test/configs/podman-preprocess-compose-test/.devcontainer/Dockerfile.in b/src/test/configs/podman-preprocess-compose-test/.devcontainer/Dockerfile.in new file mode 100644 index 000000000..4eeb54dbe --- /dev/null +++ b/src/test/configs/podman-preprocess-compose-test/.devcontainer/Dockerfile.in @@ -0,0 +1,6 @@ +#define BASE_IMAGE debian:bookworm-slim + +FROM BASE_IMAGE + +#include "common.Dockerfile" +#include "tools.Dockerfile" diff --git a/src/test/configs/podman-preprocess-compose-test/.devcontainer/bootstrap.sh b/src/test/configs/podman-preprocess-compose-test/.devcontainer/bootstrap.sh new file mode 100644 index 000000000..a587077ef --- /dev/null +++ b/src/test/configs/podman-preprocess-compose-test/.devcontainer/bootstrap.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo compose-bootstrap diff --git a/src/test/configs/podman-preprocess-compose-test/.devcontainer/common.Dockerfile b/src/test/configs/podman-preprocess-compose-test/.devcontainer/common.Dockerfile new file mode 100644 index 000000000..67d4c72e0 --- /dev/null +++ b/src/test/configs/podman-preprocess-compose-test/.devcontainer/common.Dockerfile @@ -0,0 +1 @@ +RUN apt-get update && apt-get install -y curl wget diff --git a/src/test/configs/podman-preprocess-compose-test/.devcontainer/devcontainer.json b/src/test/configs/podman-preprocess-compose-test/.devcontainer/devcontainer.json new file mode 100644 index 000000000..61d00fbb9 --- /dev/null +++ b/src/test/configs/podman-preprocess-compose-test/.devcontainer/devcontainer.json @@ -0,0 +1,7 @@ +{ + "name": "Podman Preprocess Compose Test", + "dockerComposeFile": "docker-compose.yml", + "service": "app", + "workspaceFolder": "/workspace", + "remoteUser": "root" +} diff --git a/src/test/configs/podman-preprocess-compose-test/.devcontainer/docker-compose.yml b/src/test/configs/podman-preprocess-compose-test/.devcontainer/docker-compose.yml new file mode 100644 index 000000000..c8ab5e8ed --- /dev/null +++ b/src/test/configs/podman-preprocess-compose-test/.devcontainer/docker-compose.yml @@ -0,0 +1,10 @@ +version: '3.8' + +services: + app: + build: + context: .. + dockerfile: .devcontainer/Dockerfile.in + volumes: + - ..:/workspace:cached + command: sleep infinity diff --git a/src/test/configs/podman-preprocess-compose-test/.devcontainer/tools.Dockerfile b/src/test/configs/podman-preprocess-compose-test/.devcontainer/tools.Dockerfile new file mode 100644 index 000000000..9b03a9352 --- /dev/null +++ b/src/test/configs/podman-preprocess-compose-test/.devcontainer/tools.Dockerfile @@ -0,0 +1,3 @@ +RUN apt-get update && apt-get install -y vim +COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh +RUN chmod +x /usr/local/bin/bootstrap.sh diff --git a/src/test/configs/podman-preprocess-compose-test/bootstrap.sh b/src/test/configs/podman-preprocess-compose-test/bootstrap.sh new file mode 100644 index 000000000..a587077ef --- /dev/null +++ b/src/test/configs/podman-preprocess-compose-test/bootstrap.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo compose-bootstrap diff --git a/src/test/configs/podman-preprocess-without feature/Dockerfile.in b/src/test/configs/podman-preprocess-without feature/Dockerfile.in new file mode 100644 index 000000000..4eeb54dbe --- /dev/null +++ b/src/test/configs/podman-preprocess-without feature/Dockerfile.in @@ -0,0 +1,6 @@ +#define BASE_IMAGE debian:bookworm-slim + +FROM BASE_IMAGE + +#include "common.Dockerfile" +#include "tools.Dockerfile" diff --git a/src/test/configs/podman-preprocess-without feature/bootstrap.sh b/src/test/configs/podman-preprocess-without feature/bootstrap.sh new file mode 100644 index 000000000..a9a0c42bc --- /dev/null +++ b/src/test/configs/podman-preprocess-without feature/bootstrap.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo without-feature-bootstrap diff --git a/src/test/configs/podman-preprocess-without feature/common.Dockerfile b/src/test/configs/podman-preprocess-without feature/common.Dockerfile new file mode 100644 index 000000000..67d4c72e0 --- /dev/null +++ b/src/test/configs/podman-preprocess-without feature/common.Dockerfile @@ -0,0 +1 @@ +RUN apt-get update && apt-get install -y curl wget diff --git a/src/test/configs/podman-preprocess-without feature/tools.Dockerfile b/src/test/configs/podman-preprocess-without feature/tools.Dockerfile new file mode 100644 index 000000000..92354cfbd --- /dev/null +++ b/src/test/configs/podman-preprocess-without feature/tools.Dockerfile @@ -0,0 +1,2 @@ +RUN apt-get update && apt-get install -y vim +COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh diff --git a/src/test/configs/podman-test/.devcontainer.json b/src/test/configs/podman-test/.devcontainer.json new file mode 100644 index 000000000..51c0d1bea --- /dev/null +++ b/src/test/configs/podman-test/.devcontainer.json @@ -0,0 +1,10 @@ +{ + "build": { + "dockerfile": "Dockerfile.in" + }, + "features": { + "ghcr.io/devcontainers/features/github-cli:1": { + "version": "latest" + } + } +} diff --git a/src/test/configs/podman-test/Dockerfile.in b/src/test/configs/podman-test/Dockerfile.in new file mode 100644 index 000000000..d7e4fcd28 --- /dev/null +++ b/src/test/configs/podman-test/Dockerfile.in @@ -0,0 +1,16 @@ +#define BASE_IMAGE ubuntu:20.04 +#define INSTALL_NODE +#define INSTALL_PYTHON + +FROM BASE_IMAGE + +#ifdef INSTALL_NODE +RUN apt-get update && apt-get install -y nodejs +#endif + +#ifdef INSTALL_PYTHON +RUN apt-get update && apt-get install -y python3 +#endif + +#include "common.Dockerfile" +#include "tools.Dockerfile" \ No newline at end of file diff --git a/src/test/configs/podman-test/bootstrap.sh b/src/test/configs/podman-test/bootstrap.sh new file mode 100644 index 000000000..1f4cc2df9 --- /dev/null +++ b/src/test/configs/podman-test/bootstrap.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo bootstrap diff --git a/src/test/configs/podman-test/common.Dockerfile b/src/test/configs/podman-test/common.Dockerfile new file mode 100644 index 000000000..97856b250 --- /dev/null +++ b/src/test/configs/podman-test/common.Dockerfile @@ -0,0 +1,4 @@ +RUN apt-get update && apt-get install -y curl wget + +ENV APP_ENV=development +ENV APP_DEBUG=true \ No newline at end of file diff --git a/src/test/configs/podman-test/tools.Dockerfile b/src/test/configs/podman-test/tools.Dockerfile new file mode 100644 index 000000000..8d307bce3 --- /dev/null +++ b/src/test/configs/podman-test/tools.Dockerfile @@ -0,0 +1,2 @@ +RUN apt-get update && apt-get install -y vim +COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh \ No newline at end of file diff --git a/src/test/dockerfilePreprocess.test.ts b/src/test/dockerfilePreprocess.test.ts new file mode 100644 index 000000000..97d584191 --- /dev/null +++ b/src/test/dockerfilePreprocess.test.ts @@ -0,0 +1,265 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import { assert } from 'chai'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as yaml from 'js-yaml'; +import { CLIHost } from '../spec-common/cliHost'; +import { materializeResolvedDockerfileForBuild, resolveDockerfileIncludesIfNeeded } from '../spec-node/dockerfilePreprocess'; +import { getBuildInfoForService } from '../spec-node/dockerCompose'; + +function createMockCLIHost(files: Record, platform: NodeJS.Platform = 'linux'): CLIHost { + const pathModule = platform === 'win32' ? path.win32 : path.posix; + return { + type: 'local', + platform, + arch: 'x64', + path: pathModule, + cwd: platform === 'win32' ? 'C:\\' : '/', + env: {}, + exec: () => { throw new Error('Not implemented'); }, + ptyExec: () => { throw new Error('Not implemented'); }, + homedir: async () => platform === 'win32' ? 'C:\\Users\\test' : '/home/test', + tmpdir: async () => platform === 'win32' ? 'C:\\tmp' : '/tmp', + isFile: async (filepath: string) => filepath in files, + isFolder: async () => false, + readFile: async (filepath: string) => { + if (!(filepath in files)) { + throw new Error(`File not found: ${filepath}`); + } + return Buffer.from(files[filepath]); + }, + writeFile: async (filepath: string, content: Buffer) => { + files[filepath] = content.toString(); + }, + deleteFile: async (filepath: string) => { + delete files[filepath]; + }, + rename: async () => { }, + mkdirp: async () => { }, + readDir: async () => [], + getUsername: async () => 'test', + toCommonURI: async () => undefined, + connect: () => { throw new Error('Not implemented'); }, + }; +} + +describe('resolveDockerfileIncludesIfNeeded', () => { + it('returns source Dockerfile unchanged when not using .in extension', async () => { + const files: Record = { + '/workspace/Dockerfile': 'FROM debian:latest\nRUN echo ok', + }; + const cliHost = createMockCLIHost(files); + const result = await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/Dockerfile'); + assert.isFalse(result.preprocessed); + assert.equal(result.effectiveDockerfileContent, files['/workspace/Dockerfile']); + }); + + it('expands #include lines for .in files without writing a generated Dockerfile yet', async () => { + const podmanTestConfigPath = path.resolve(__dirname, 'configs', 'podman-test'); + const sourceDockerfilePath = path.join(podmanTestConfigPath, 'Dockerfile.in'); + const commonDockerfilePath = path.join(podmanTestConfigPath, 'common.Dockerfile'); + const includedDockerfilePath = path.join(podmanTestConfigPath, 'tools.Dockerfile'); + const copiedFilePath = path.join(podmanTestConfigPath, 'bootstrap.sh'); + const sourceDockerfileContent = fs.readFileSync(sourceDockerfilePath).toString(); + const commonDockerfileContent = fs.readFileSync(commonDockerfilePath).toString(); + const includedDockerfileContent = fs.readFileSync(includedDockerfilePath).toString(); + const copiedFileContent = fs.readFileSync(copiedFilePath).toString(); + const files: Record = { + [sourceDockerfilePath]: sourceDockerfileContent, + [commonDockerfilePath]: commonDockerfileContent, + [includedDockerfilePath]: includedDockerfileContent, + [copiedFilePath]: copiedFileContent, + }; + const cliHost = createMockCLIHost(files); + const result = await resolveDockerfileIncludesIfNeeded(cliHost, sourceDockerfilePath); + assert.isTrue(result.preprocessed); + assert.equal( + result.effectiveDockerfileContent, + '\nFROM ubuntu:20.04\n\nRUN apt-get update && apt-get install -y nodejs\n\nRUN apt-get update && apt-get install -y python3\n\nRUN apt-get update && apt-get install -y curl wget\n\nENV APP_ENV=development\nENV APP_DEBUG=true\nRUN apt-get update && apt-get install -y vim\nCOPY ./bootstrap.sh /usr/local/bin/bootstrap.sh' + ); + assert.deepEqual(Object.keys(files).sort(), [sourceDockerfilePath, commonDockerfilePath, includedDockerfilePath, copiedFilePath].sort()); + }); + + it('materializes a preprocessed Dockerfile in the source directory and cleans it up', async () => { + const files: Record = { + '/workspace/.devcontainer/Dockerfile.in': [ + 'FROM docker.io/debian:latest', + '#include "scripts.Dockerfile"', + ].join('\n'), + '/workspace/.devcontainer/scripts.Dockerfile': [ + 'RUN echo preparing scripts', + 'COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh', + ].join('\n'), + '/workspace/.devcontainer/bootstrap.sh': '#!/bin/sh\necho bootstrap', + }; + const cliHost = createMockCLIHost(files); + const result = await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/.devcontainer/Dockerfile.in'); + const materialized = await materializeResolvedDockerfileForBuild(cliHost, result); + + assert.isTrue(result.preprocessed); + assert.equal(path.dirname(materialized.dockerfilePath), '/workspace/.devcontainer'); + assert.include(result.effectiveDockerfileContent, 'RUN echo preparing scripts'); + assert.include(result.effectiveDockerfileContent, 'COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh'); + assert.equal(files[materialized.dockerfilePath], result.effectiveDockerfileContent); + await materialized.dispose(); + assert.notProperty(files, materialized.dockerfilePath); + }); + + it('materializes compose-referenced preprocessed Dockerfiles beside the original Dockerfile to preserve relative COPY paths', async () => { + const composeFixturePath = path.resolve(__dirname, 'configs', 'podman-preprocess-compose-test'); + const composeFilePath = path.join(composeFixturePath, '.devcontainer', 'docker-compose.yml'); + const dockerfilePath = path.join(composeFixturePath, '.devcontainer', 'Dockerfile.in'); + const commonDockerfilePath = path.join(composeFixturePath, '.devcontainer', 'common.Dockerfile'); + const toolsDockerfilePath = path.join(composeFixturePath, '.devcontainer', 'tools.Dockerfile'); + const copiedFilePath = path.join(composeFixturePath, '.devcontainer', 'bootstrap.sh'); + const composeContent = fs.readFileSync(composeFilePath, 'utf8'); + const files: Record = { + [composeFilePath]: composeContent, + [dockerfilePath]: fs.readFileSync(dockerfilePath, 'utf8'), + [commonDockerfilePath]: fs.readFileSync(commonDockerfilePath, 'utf8'), + [toolsDockerfilePath]: fs.readFileSync(toolsDockerfilePath, 'utf8'), + [copiedFilePath]: fs.readFileSync(copiedFilePath, 'utf8'), + }; + const cliHost = createMockCLIHost(files); + const composeConfig = yaml.load(composeContent) as any; + const serviceInfo = getBuildInfoForService(composeConfig.services.app, cliHost.path, [composeFilePath]); + + assert.isDefined(serviceInfo.build); + serviceInfo.build!.context = cliHost.path.resolve(path.dirname(composeFilePath), serviceInfo.build!.context); + const resolvedDockerfilePath = cliHost.path.isAbsolute(serviceInfo.build!.dockerfilePath) + ? serviceInfo.build!.dockerfilePath + : cliHost.path.resolve(serviceInfo.build!.context, serviceInfo.build!.dockerfilePath); + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, resolvedDockerfilePath); + const materialized = await materializeResolvedDockerfileForBuild(cliHost, resolvedDockerfile); + + assert.equal(resolvedDockerfilePath, dockerfilePath); + assert.equal(path.dirname(materialized.dockerfilePath), path.dirname(dockerfilePath)); + assert.include(files[materialized.dockerfilePath], 'COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh'); + assert.equal(path.resolve(path.dirname(materialized.dockerfilePath), 'bootstrap.sh'), copiedFilePath); + + await materialized.dispose(); + assert.notProperty(files, materialized.dockerfilePath); + }); + + it('expands includes for podman-preprocess-without feature without writing a generated Dockerfile yet', async () => { + const fixturePath = path.resolve(__dirname, 'configs', 'podman-preprocess-without feature'); + const sourceDockerfilePath = path.join(fixturePath, 'Dockerfile.in'); + const commonDockerfilePath = path.join(fixturePath, 'common.Dockerfile'); + const toolsDockerfilePath = path.join(fixturePath, 'tools.Dockerfile'); + const copiedFilePath = path.join(fixturePath, 'bootstrap.sh'); + const files: Record = { + [sourceDockerfilePath]: fs.readFileSync(sourceDockerfilePath, 'utf8'), + [commonDockerfilePath]: fs.readFileSync(commonDockerfilePath, 'utf8'), + [toolsDockerfilePath]: fs.readFileSync(toolsDockerfilePath, 'utf8'), + [copiedFilePath]: fs.readFileSync(copiedFilePath, 'utf8'), + }; + const cliHost = createMockCLIHost(files); + + const result = await resolveDockerfileIncludesIfNeeded(cliHost, sourceDockerfilePath); + + assert.isTrue(result.preprocessed); + assert.include(result.effectiveDockerfileContent, 'FROM debian:bookworm-slim'); + assert.include(result.effectiveDockerfileContent, 'RUN apt-get update && apt-get install -y curl wget'); + assert.include(result.effectiveDockerfileContent, 'COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh'); + assert.deepEqual(Object.keys(files).sort(), [sourceDockerfilePath, commonDockerfilePath, toolsDockerfilePath, copiedFilePath].sort()); + }); + + it('materializes podman-preprocess-without feature beside the original Dockerfile so relative COPY keeps working', async () => { + const fixturePath = path.resolve(__dirname, 'configs', 'podman-preprocess-without feature'); + const sourceDockerfilePath = path.join(fixturePath, 'Dockerfile.in'); + const commonDockerfilePath = path.join(fixturePath, 'common.Dockerfile'); + const toolsDockerfilePath = path.join(fixturePath, 'tools.Dockerfile'); + const copiedFilePath = path.join(fixturePath, 'bootstrap.sh'); + const files: Record = { + [sourceDockerfilePath]: fs.readFileSync(sourceDockerfilePath, 'utf8'), + [commonDockerfilePath]: fs.readFileSync(commonDockerfilePath, 'utf8'), + [toolsDockerfilePath]: fs.readFileSync(toolsDockerfilePath, 'utf8'), + [copiedFilePath]: fs.readFileSync(copiedFilePath, 'utf8'), + }; + const cliHost = createMockCLIHost(files); + + const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, sourceDockerfilePath); + const materialized = await materializeResolvedDockerfileForBuild(cliHost, resolvedDockerfile); + + assert.equal(path.dirname(materialized.dockerfilePath), fixturePath); + assert.include(files[materialized.dockerfilePath], 'COPY ./bootstrap.sh /usr/local/bin/bootstrap.sh'); + assert.equal(path.resolve(path.dirname(materialized.dockerfilePath), 'bootstrap.sh'), copiedFilePath); + + await materialized.dispose(); + assert.notProperty(files, materialized.dockerfilePath); + assert.property(files, copiedFilePath); + }); + + it('returns the original Dockerfile path when materialization is unnecessary', async () => { + const files: Record = { + '/workspace/Dockerfile': 'FROM debian:latest\nRUN echo ok', + }; + const cliHost = createMockCLIHost(files); + const result = await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/Dockerfile'); + const materialized = await materializeResolvedDockerfileForBuild(cliHost, result); + + assert.equal(materialized.dockerfilePath, '/workspace/Dockerfile'); + await materialized.dispose(); + assert.equal(files['/workspace/Dockerfile'], 'FROM debian:latest\nRUN echo ok'); + }); + + it('fails with a clear error when #include has a cycle', async () => { + const files: Record = { + '/workspace/a.Dockerfile.in': '#include "b.Dockerfile"\nRUN echo a', + '/workspace/b.Dockerfile': '#include "a.Dockerfile.in"\nRUN echo b', + }; + const cliHost = createMockCLIHost(files); + let err: any; + try { + await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/a.Dockerfile.in'); + } catch (e) { + err = e; + } + assert.ok(err); + assert.include(String(err.message || err), 'Cyclic #include detected while preprocessing Dockerfile'); + }); + + it('supports #define/#undef with conditionals and #warning', async () => { + const files: Record = { + '/workspace/Dockerfile.in': [ + '#define BASE_IMAGE docker.io/debian:bookworm', + '#if defined(BASE_IMAGE)', + 'FROM BASE_IMAGE', + '#else', + '#error BASE_IMAGE must be defined', + '#endif', + '#warning Using BASE_IMAGE', + '#undef BASE_IMAGE', + '#ifndef BASE_IMAGE', + 'RUN echo fallback-ok', + '#endif', + ].join('\n'), + }; + const cliHost = createMockCLIHost(files); + const result = await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/Dockerfile.in'); + + assert.isTrue(result.preprocessed); + assert.include(result.effectiveDockerfileContent, 'FROM docker.io/debian:bookworm'); + assert.include(result.effectiveDockerfileContent, '# warning: Using docker.io/debian:bookworm'); + assert.include(result.effectiveDockerfileContent, 'RUN echo fallback-ok'); + }); + + it('fails with a clear error when preprocessed output has no resolved FROM', async () => { + const files: Record = { + '/workspace/Dockerfile.in': '#if 0\nFROM docker.io/debian:latest\n#endif\nRUN echo missing-from', + }; + const cliHost = createMockCLIHost(files); + let err: any; + try { + await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/Dockerfile.in'); + } catch (e) { + err = e; + } + + assert.ok(err); + assert.include(String(err.message || err), 'contains no resolved FROM instruction'); + }); +});