From 11c0554daeea5f2028fc185ba5446f35e096658e Mon Sep 17 00:00:00 2001 From: Michael O'Cleirigh Date: Fri, 4 Jul 2014 15:31:52 -0400 Subject: [PATCH 1/3] Add support for environment variable substition for ADD and COPY resources Supports the case where the name of the file is supposed to be resolved using an environment variable. ENV VAR sub ADD files/$VAR.dat /$VAR.dat ADD files/${VAR}.dat /${VAR}.dat In both cases the VAR is substituted for sub. This allows the tar file creation process to capture the real source file. --- .../client/command/BuildImgCmd.java | 67 +++++++++++++++++-- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/github/dockerjava/client/command/BuildImgCmd.java b/src/main/java/com/github/dockerjava/client/command/BuildImgCmd.java index 76039a33b..6070fbb92 100644 --- a/src/main/java/com/github/dockerjava/client/command/BuildImgCmd.java +++ b/src/main/java/com/github/dockerjava/client/command/BuildImgCmd.java @@ -6,7 +6,9 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -35,7 +37,9 @@ public class BuildImgCmd extends AbstrDockerCmd { private static final Logger LOGGER = LoggerFactory.getLogger(BuildImgCmd.class); - private static final Pattern ADD_PATTERN = Pattern.compile("^ADD\\s+(.*)\\s+(.*)$"); + private static final Pattern ADD_OR_COPY_PATTERN = Pattern.compile("^(ADD|COPY)\\s+(.*)\\s+(.*)$"); + + private static final Pattern ENV_PATTERN = Pattern.compile("^ENV\\s+(.*)\\s+(.*)$"); private File dockerFolder = null; private InputStream tarInputStream = null; @@ -133,14 +137,40 @@ protected File buildDockerFolderTar() { List filesToAdd = new ArrayList(); filesToAdd.add(dockerFile); + MapenvironmentMap = new HashMap(); + + int lineNumber = 0; + for (String cmd : dockerFileContent) { - final Matcher matcher = ADD_PATTERN.matcher(cmd.trim()); + + lineNumber++; + + if (cmd.trim().isEmpty() || cmd.startsWith("#")) + continue; // skip emtpy and commend lines + + final Matcher envMatcher = ENV_PATTERN.matcher(cmd.trim()); + + if (envMatcher.find()) { + if (envMatcher.groupCount() != 2) + throw new DockerException(String.format("Wrong ENV format on line [%d]", lineNumber)); + + String variable = envMatcher.group(1).trim(); + + String value = envMatcher.group(2).trim(); + + environmentMap.put(variable, value); + } + + + final Matcher matcher = ADD_OR_COPY_PATTERN.matcher(cmd.trim()); if (matcher.find()) { - if (matcher.groupCount() != 2) { - throw new DockerException(String.format("Wrong format on line [%s]", cmd)); + if (matcher.groupCount() != 3) { + throw new DockerException(String.format("Wrong ADD or COPY format on line [%d]", lineNumber)); } - String resource = matcher.group(1).trim(); + String extractedResource = matcher.group(2); + + String resource = filterForEnvironmentVars(extractedResource, environmentMap); if(isFileResource(resource)) { File src = new File(resource); @@ -170,6 +200,33 @@ protected File buildDockerFolderTar() { } } + private String filterForEnvironmentVars(String extractedResource, + Map environmentMap) { + + if (environmentMap.size() > 0) { + + String currentResourceContent = extractedResource; + + for (Map.Entry entry : environmentMap.entrySet()) { + + String variable = entry.getKey(); + + String replacementValue = entry.getValue(); + + // handle: $VARIABLE case + currentResourceContent = currentResourceContent.replaceAll("\\$" + variable, replacementValue); + + // handle ${VARIABLE} case + currentResourceContent = currentResourceContent.replaceAll("\\$\\{" + variable + "\\}", replacementValue); + + } + + return currentResourceContent; + } + else + return extractedResource; + } + private static boolean isFileResource(String resource) { URI uri; try { From 5cbefec285357443230ae5a663f8b60034bee4ab Mon Sep 17 00:00:00 2001 From: Michael O'Cleirigh Date: Mon, 7 Jul 2014 13:41:01 -0400 Subject: [PATCH 2/3] Add extra parameters to the DockerClient constructor's The defaults remain 10 seconds for the read time out and to enable the LoggingFilter. Allows the caller to change the read time out and to disable the LoggingFilter For large images the logging filter spam's the console with the binary data that is being transferred. This new setting allows it to be turned off. --- .../dockerjava/client/DockerClient.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/github/dockerjava/client/DockerClient.java b/src/main/java/com/github/dockerjava/client/DockerClient.java index 0dc927f64..deb6ebedf 100644 --- a/src/main/java/com/github/dockerjava/client/DockerClient.java +++ b/src/main/java/com/github/dockerjava/client/DockerClient.java @@ -67,13 +67,20 @@ public class DockerClient { private AuthConfig authConfig; public DockerClient() throws DockerException { - this(Config.createConfig()); + this(10000, true); + } + public DockerClient(Integer readTimeout, boolean enableLoggingFilter) throws DockerException { + this(Config.createConfig(), readTimeout, enableLoggingFilter); } public DockerClient(String serverUrl) throws DockerException { - this(configWithServerUrl(serverUrl)); + this(serverUrl, 10000, true); } + public DockerClient(String serverUrl, Integer readTimeout, boolean enableLoggingFilter) throws DockerException { + this(configWithServerUrl(serverUrl), readTimeout, enableLoggingFilter); + } + private static Config configWithServerUrl(String serverUrl) throws DockerException { final Config c = Config.createConfig(); @@ -81,7 +88,7 @@ private static Config configWithServerUrl(String serverUrl) return c; } - private DockerClient(Config config) { + public DockerClient(Config config, Integer readTimeout, boolean enableLoggingFilter) { ClientConfig clientConfig = new DefaultClientConfig(); SchemeRegistry schemeRegistry = new SchemeRegistry(); @@ -101,14 +108,19 @@ private DockerClient(Config config) { client = new ApacheHttpClient4(new ApacheHttpClient4Handler(httpClient, null, false), clientConfig); - client.setReadTimeout(10000); + // 1 hour + client.setReadTimeout(readTimeout); client.addFilter(new JsonClientFilter()); - client.addFilter(new SelectiveLoggingFilter()); + + if (enableLoggingFilter) + client.addFilter(new SelectiveLoggingFilter()); baseResource = client.resource(config.url + "/v" + config.version); } + + public void setCredentials(String username, String password, String email) { if (username == null) { throw new IllegalArgumentException("username is null"); From e6666f8b77c5df609d61999e96f8cc74772bf483 Mon Sep 17 00:00:00 2001 From: Michael O'Cleirigh Date: Mon, 7 Jul 2014 14:37:09 -0400 Subject: [PATCH 3/3] Add new unit test case for ENV substitution --- .../client/command/BuildImageCmdTest.java | 11 ++++++++ .../resources/testENVSubstitution/Dockerfile | 11 ++++++++ .../subst-file-2-abc123.txt | 1 + .../testENVSubstitution/subst-file-abc123.txt | 1 + .../resources/testENVSubstitution/testrun.sh | 26 +++++++++++++++++++ 5 files changed, 50 insertions(+) create mode 100644 src/test/resources/testENVSubstitution/Dockerfile create mode 100644 src/test/resources/testENVSubstitution/subst-file-2-abc123.txt create mode 100644 src/test/resources/testENVSubstitution/subst-file-abc123.txt create mode 100644 src/test/resources/testENVSubstitution/testrun.sh diff --git a/src/test/java/com/github/dockerjava/client/command/BuildImageCmdTest.java b/src/test/java/com/github/dockerjava/client/command/BuildImageCmdTest.java index 803835ad3..6fb5cbccd 100644 --- a/src/test/java/com/github/dockerjava/client/command/BuildImageCmdTest.java +++ b/src/test/java/com/github/dockerjava/client/command/BuildImageCmdTest.java @@ -12,6 +12,8 @@ import java.io.IOException; import java.io.StringWriter; import java.lang.reflect.Method; +import java.net.URI; +import java.net.URISyntaxException; import org.apache.commons.io.IOUtils; import org.apache.commons.io.LineIterator; @@ -113,6 +115,7 @@ public void testDockerBuilderAddFolder() throws DockerException, dockerfileBuild(baseDir, "Successfully executed testAddFolder.sh"); } + private String dockerfileBuild(File baseDir, String expectedText) throws DockerException, IOException { @@ -222,4 +225,12 @@ public void testNetCatDockerfileBuilder() throws DockerException, dockerClient.stopContainerCmd(container.getId()).withTimeout(0).exec(); } + + @Test + public void testAddAndCopySubstitution () throws DockerException, IOException { + File baseDir = new File(Thread.currentThread().getContextClassLoader() + .getResource("testENVSubstitution").getFile()); + dockerfileBuild(baseDir, "Successfully executed testAddFolder.sh"); + + } } \ No newline at end of file diff --git a/src/test/resources/testENVSubstitution/Dockerfile b/src/test/resources/testENVSubstitution/Dockerfile new file mode 100644 index 000000000..de06ddbba --- /dev/null +++ b/src/test/resources/testENVSubstitution/Dockerfile @@ -0,0 +1,11 @@ +FROM ubuntu + +# Copy testrun.sh files into the container + +ENV variable abc123 +ADD ./testrun.sh /tmp/ +ADD ./subst-file-$variable.txt /tmp/ +COPY ./subst-file-2-${variable}.txt /tmp/ +RUN cp /tmp/testrun.sh /usr/local/bin/ && chmod +x /usr/local/bin/testrun.sh + +CMD ["testrun.sh"] diff --git a/src/test/resources/testENVSubstitution/subst-file-2-abc123.txt b/src/test/resources/testENVSubstitution/subst-file-2-abc123.txt new file mode 100644 index 000000000..ac1acee1e --- /dev/null +++ b/src/test/resources/testENVSubstitution/subst-file-2-abc123.txt @@ -0,0 +1 @@ +Old File \ No newline at end of file diff --git a/src/test/resources/testENVSubstitution/subst-file-abc123.txt b/src/test/resources/testENVSubstitution/subst-file-abc123.txt new file mode 100644 index 000000000..ac1acee1e --- /dev/null +++ b/src/test/resources/testENVSubstitution/subst-file-abc123.txt @@ -0,0 +1 @@ +Old File \ No newline at end of file diff --git a/src/test/resources/testENVSubstitution/testrun.sh b/src/test/resources/testENVSubstitution/testrun.sh new file mode 100644 index 000000000..90a416a1c --- /dev/null +++ b/src/test/resources/testENVSubstitution/testrun.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +echo "Hello Word" > hello.txt +echo "hello.txt Created" + +TEST_FILE=/tmp/subst-file-abc123.txt +TEST_FILE_2=/tmp/subst-file-2-abc123.txt + +if test -f $TEST_FILE +then + + if test -f $TEST_FILE_2 + then + exit 0 + else + echo "$TEST_FILE_2 does not exist" + exit 1 + fi + +else + + echo "$TEST_FILE does not exist" + exit 1 + +fi +