From 403c8964de8b1ec123c23ccef95b2a449b13ab16 Mon Sep 17 00:00:00 2001 From: gauravsaini04 <147703805+gauravsaini04@users.noreply.github.com> Date: Fri, 23 Feb 2024 05:27:16 +0530 Subject: [PATCH 001/247] [ docker-in-docker ] - buildx can fallback to previous version if latest artifact not found (#869) * [ docker-in-docker ] - buildx fallback to prev. version * corrected the command inside get_previous_version fn to get the second to latest version instead * changes in response to comments by @samruddhikhandale * minor change * changes as requested in comments by @samruddhikhandale --- .../devcontainer-feature.json | 2 +- src/docker-in-docker/install.sh | 30 +++++- .../docker_build_fallback_buildx.sh | 95 +++++++++++++++++++ test/docker-in-docker/scenarios.json | 11 +++ 4 files changed, 133 insertions(+), 5 deletions(-) create mode 100644 test/docker-in-docker/docker_build_fallback_buildx.sh diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index 744bc1d81..5c62a057d 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.9.0", + "version": "2.9.1", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index db3cefc6a..8943c91a6 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -139,7 +139,7 @@ else fi # Install dependencies -check_packages apt-transport-https curl ca-certificates pigz iptables gnupg2 dirmngr wget +check_packages apt-transport-https curl ca-certificates pigz iptables gnupg2 dirmngr wget jq if ! type git > /dev/null 2>&1; then check_packages git fi @@ -332,14 +332,36 @@ fi usermod -aG docker ${USERNAME} +# Function to fetch the version released prior to the latest version +get_previous_version() { + repo_url=$1 + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects +} + +install_previous_version_artifacts() { + wget_exit_code=$? + if [ $wget_exit_code -eq 8 ]; then # failure due to 404: Not Found. + echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx v${buildx_version}..." + repo_url="https://api.github.com/repos/docker/buildx/releases" # GitHub repository URL + previous_version=$(get_previous_version "${repo_url}") + buildx_file_name="buildx-${previous_version}.linux-${architecture}" + echo -e "\nAttempting to install ${previous_version}" + wget https://github.com/docker/buildx/releases/download/${previous_version}/${buildx_file_name} + else + echo "(!) Failed to download docker buildx with exit code: $wget_exit_code" + exit 1 + fi +} + if [ "${INSTALL_DOCKER_BUILDX}" = "true" ]; then buildx_version="latest" find_version_from_git_tags buildx_version "https://github.com/docker/buildx" "refs/tags/v" - echo "(*) Installing buildx ${buildx_version}..." buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" - cd /tmp && wget "https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name}" - + + cd /tmp + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} || install_previous_version_artifacts + docker_home="/usr/libexec/docker" cli_plugins_dir="${docker_home}/cli-plugins" diff --git a/test/docker-in-docker/docker_build_fallback_buildx.sh b/test/docker-in-docker/docker_build_fallback_buildx.sh new file mode 100644 index 000000000..8d30d2a46 --- /dev/null +++ b/test/docker-in-docker/docker_build_fallback_buildx.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests before test for fallback +HL="\033[1;33m" +N="\033[0;37m" +echo -e "\n๐Ÿ‘‰${HL} docker/buildx version as installed by docker-in-docker feature${N}" +check "docker-buildx" docker buildx version +check "docker-build" docker build ./ +check "docker-buildx" bash -c "docker buildx version" +check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" + +echo -e "\n๐Ÿ‘‰${HL} Creating a scenario for fallback${N}\n" +# Code to test the made up scenario when latest version of docker/buildx fails on wget command for fetching the artifacts +repo_url="https://api.github.com/repos/docker/buildx/releases" # GitHub repository URL +architecture="$(dpkg --print-architecture)" + +# Function to fetch the latest version of the plugin +get_latest_version() { + curl -s "$repo_url/latest" | jq -r '.tag_name' +} + +# Function to fetch the previous version of the plugin +get_previous_version() { + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects +} + +# Function to change the patch number in a semver version +change_patch_number() { + local version="$1" # Input version + local new_patch="$2" # New patch number + # Extract major, minor, and current patch numbers + local major=$(echo "$version" | cut -d. -f1) + local minor=$(echo "$version" | cut -d. -f2) + local current_patch=$(echo "$version" | cut -d. -f3) + # Construct the new version with the updated patch number + local new_version="$major.$minor.$new_patch" + echo "$new_version" +} + +change_version_to_fail() { + latest_version=$1 + new_patch_number="xyz" # for testing a tag not found scenario for docker/buildx plugin + latest_version=$(get_latest_version) # can take latest_version from fn get_latest_version + buildx_version_fallback_test=$(change_patch_number "$latest_version" "$new_patch_number") # for testing a tag not found scenario for docker/buildx plugin + echo "${buildx_version_fallback_test}" +} + +install_previous_version_artifacts() { + wget_exit_code=$? + if [ $wget_exit_code -ne 0 ]; then # means wget command to fetch latest version failed + if [ $wget_exit_code -eq 8 ]; then # failure due to 404: Not Found. + echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx ${buildx_version}..." + previous_version=$(get_previous_version) + echo -e "\nAttempting to install ${previous_version}" + buildx_file_name="buildx-${previous_version}.linux-${architecture}" + wget https://github.com/docker/buildx/releases/download/${previous_version}/${buildx_file_name} + else + echo "(!) Failed to download docker buildx with exit code: $wget_exit_code" + exit 1 + fi + fi +} + +test_version=$(change_version_to_fail "$(get_latest_version)") +buildx_file_name="buildx-${test_version}.linux-${architecture}" +buildx_version=$test_version + +# This wget command will fail as the wrong version won't fetch artifact +wget https://github.com/docker/buildx/releases/download/${buildx_version}/${buildx_file_name} || install_previous_version_artifacts + +docker_home="/usr/libexec/docker" +cli_plugins_dir="${docker_home}/cli-plugins" + +mkdir -p ${cli_plugins_dir} +mv ${buildx_file_name} ${cli_plugins_dir}/docker-buildx +chmod +x ${cli_plugins_dir}/docker-buildx + +chown -R "${USERNAME}:docker" "${docker_home}" +chmod -R g+r+w "${docker_home}" +find "${docker_home}" -type d -print0 | xargs -n 1 -0 chmod g+s + +# Definition specific tests after test for fallback +echo -e "\n๐Ÿ‘‰${HL} docker/buildx version as installed by test for fallback${N}" +check "docker-buildx" docker buildx version +check "docker-build" docker build ./ +check "docker-buildx" bash -c "docker buildx version" +check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" + +# Report result +reportResults diff --git a/test/docker-in-docker/scenarios.json b/test/docker-in-docker/scenarios.json index 6de695555..df27d6c28 100644 --- a/test/docker-in-docker/scenarios.json +++ b/test/docker-in-docker/scenarios.json @@ -87,6 +87,17 @@ } } }, + "docker_build_fallback_buildx": { + "image": "ubuntu:focal", + "features": { + "docker-in-docker": { + "version": "latest", + "installDockerBuildx": true, + "moby": "false", + "dockerDashComposeVersion": "v2" + } + } + }, // DO NOT REMOVE: This scenario is used by the docker-in-docker-stress-test workflow "docker_with_on_create_command": { "image": "mcr.microsoft.com/devcontainers/base:debian", From c04ead01d2dd154dc782d3bb610e08e6d740b77c Mon Sep 17 00:00:00 2001 From: Nebula <40148908+nebula-it@users.noreply.github.com> Date: Thu, 22 Feb 2024 16:51:04 -0800 Subject: [PATCH 002/247] Add support for including PowerShell profile (#804) * Add support for including PowerShell profile * Update version in devcontainer-feature.json * Add powershellProfileURL to options * Update src/powershell/devcontainer-feature.json Co-authored-by: Samruddhi Khandale * Update README.md * Create testProfile.ps1 Added a Profile for testing * Add powershell profile test * Update install_modules.sh * Undo README.md changes * Update scenarios.json * Delete test/powershell/testProfile.ps1 --------- Co-authored-by: Samruddhi Khandale Co-authored-by: Samruddhi Khandale --- src/powershell/devcontainer-feature.json | 9 +++++++-- src/powershell/install.sh | 7 +++++++ test/powershell/install_modules.sh | 1 + test/powershell/scenarios.json | 3 ++- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index c72923fdc..0c15b6888 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.2.0", + "version": "1.3.0", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", @@ -19,6 +19,11 @@ "type": "string", "default": "", "description": "Optional comma separated list of PowerShell modules to install." + }, + "powershellProfileURL ": { + "type": "string", + "default": "", + "description": "Optional (publicly accessible) URL to download PowerShell profile." } }, "customizations": { @@ -31,4 +36,4 @@ "installsAfter": [ "ghcr.io/devcontainers/features/common-utils" ] -} \ No newline at end of file +} diff --git a/src/powershell/install.sh b/src/powershell/install.sh index 666071f55..dadbc306f 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -14,6 +14,7 @@ rm -rf /var/lib/apt/lists/* POWERSHELL_VERSION=${VERSION:-"latest"} POWERSHELL_MODULES="${MODULES}" +POWERSHELL_PROFILE_URL="${PROFILE_URL}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" POWERSHELL_ARCHIVE_ARCHITECTURES="amd64" @@ -162,6 +163,12 @@ if [ ${#POWERSHELL_MODULES[@]} -gt 0 ]; then done fi +# If URL for powershell profile is provided, download it to '/opt/microsoft/powershell/7/profile.ps1' +if [ -n "$POWERSHELL_PROFILE_URL" ]; then + echo "Downloading PowerShell Profile from: $POWERSHELL_PROFILE_URL" + curl -sSL -o "/opt/microsoft/powershell/7/profile.ps1" "$POWERSHELL_PROFILE_URL" +fi + # Clean up rm -rf /var/lib/apt/lists/* diff --git a/test/powershell/install_modules.sh b/test/powershell/install_modules.sh index 0bb0172e6..1415af2c1 100644 --- a/test/powershell/install_modules.sh +++ b/test/powershell/install_modules.sh @@ -8,6 +8,7 @@ source dev-container-features-test-lib # Extension-specific tests check "az.resources" pwsh -Command "(Get-Module -ListAvailable -Name Az.Resources).Version.ToString()" check "az.storage" pwsh -Command "(Get-Module -ListAvailable -Name Az.Storage).Version.ToString()" +check "profile" pwsh -Command "(Get-Variable $env:ProfileLoaded).Value" # Report result reportResults diff --git a/test/powershell/scenarios.json b/test/powershell/scenarios.json index 8ced96498..b2659d7ad 100644 --- a/test/powershell/scenarios.json +++ b/test/powershell/scenarios.json @@ -3,7 +3,8 @@ "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { "powershell": { - "modules": "az.resources, az.storage" + "modules": "az.resources, az.storage", + "powershellProfileURL": "https://raw.githubusercontent.com/codspace/powershell-profile/main/Test-Profile.ps1" } } } From f4f492ce2b9c11afc441870424aef560de4f092c Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Thu, 22 Feb 2024 16:57:10 -0800 Subject: [PATCH 003/247] Automated documentation update (#874) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/powershell/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/powershell/README.md b/src/powershell/README.md index 6112de79e..a44614604 100644 --- a/src/powershell/README.md +++ b/src/powershell/README.md @@ -17,6 +17,7 @@ Installs PowerShell along with needed dependencies. Useful for base Dockerfiles |-----|-----|-----|-----| | version | Select or enter a version of PowerShell. | string | latest | | modules | Optional comma separated list of PowerShell modules to install. | string | - | +| powershellProfileURL | Optional (publicly accessible) URL to download PowerShell profile. | string | - | ## Customizations From 758b0322d412df9d43274a4ac882e312d56a2d1d Mon Sep 17 00:00:00 2001 From: Lucas Fernando Cardoso Nunes Date: Fri, 23 Feb 2024 16:42:24 -0300 Subject: [PATCH 004/247] sync `install.sh` to `devcontainer-feature.json` default values (#867) * add default value to MOBYBUILDXVERSION Signed-off-by: Lucas Fernando Cardoso Nunes * bump updated features patch versions Signed-off-by: Lucas Fernando Cardoso Nunes * sync missing default values to `install.sh` find `^[A-Z0-9_]+="\$\{[^:]+?\}` on `install.sh` files Signed-off-by: Lucas Fernando Cardoso Nunes --------- Signed-off-by: Lucas Fernando Cardoso Nunes --- src/docker-in-docker/install.sh | 4 ++-- src/docker-outside-of-docker/devcontainer-feature.json | 2 +- src/docker-outside-of-docker/install.sh | 2 +- src/dotnet/devcontainer-feature.json | 2 +- src/dotnet/install.sh | 6 +++--- src/java/devcontainer-feature.json | 2 +- src/java/install.sh | 2 +- src/powershell/devcontainer-feature.json | 2 +- src/powershell/install.sh | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index 8943c91a6..889d1dbb2 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -10,10 +10,10 @@ DOCKER_VERSION="${VERSION:-"latest"}" # The Docker/Moby Engine + CLI should match in version USE_MOBY="${MOBY:-"true"}" -MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION}" +MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"0.12.0"}" DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v1"}" # v1 or v2 or none AZURE_DNS_AUTO_DETECTION="${AZUREDNSAUTODETECTION:-"true"}" -DOCKER_DEFAULT_ADDRESS_POOL="${DOCKERDEFAULTADDRESSPOOL}" +DOCKER_DEFAULT_ADDRESS_POOL="${DOCKERDEFAULTADDRESSPOOL:-""}" USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" INSTALL_DOCKER_BUILDX="${INSTALLDOCKERBUILDX:-"true"}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index 78ad1162c..a17757c06 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.4.0", + "version": "1.4.1", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index 988b7e43c..da936235c 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -9,7 +9,7 @@ DOCKER_VERSION="${VERSION:-"latest"}" USE_MOBY="${MOBY:-"true"}" -MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION}" +MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"0.12.0"}" DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v1"}" # v1 or v2 or none ENABLE_NONROOT_DOCKER="${ENABLE_NONROOT_DOCKER:-"true"}" diff --git a/src/dotnet/devcontainer-feature.json b/src/dotnet/devcontainer-feature.json index 878a95d3f..78a061d23 100644 --- a/src/dotnet/devcontainer-feature.json +++ b/src/dotnet/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "dotnet", - "version": "2.0.4", + "version": "2.0.5", "name": "Dotnet CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/dotnet", "description": "This Feature installs the latest .NET SDK, which includes the .NET CLI and the shared runtime. Options are provided to choose a different version or additional versions.", diff --git a/src/dotnet/install.sh b/src/dotnet/install.sh index ff9244d3a..237a8a0be 100644 --- a/src/dotnet/install.sh +++ b/src/dotnet/install.sh @@ -7,9 +7,9 @@ # Docs: https://github.com/devcontainers/features/tree/main/src/dotnet # Maintainer: The Dev Container spec maintainers DOTNET_VERSION="${VERSION:-"latest"}" -ADDITIONAL_VERSIONS="${ADDITIONALVERSIONS}" -DOTNET_RUNTIME_VERSIONS="${DOTNETRUNTIMEVERSIONS}" -ASPNETCORE_RUNTIME_VERSIONS="${ASPNETCORERUNTIMEVERSIONS}" +ADDITIONAL_VERSIONS="${ADDITIONALVERSIONS:-""}" +DOTNET_RUNTIME_VERSIONS="${DOTNETRUNTIMEVERSIONS:-""}" +ASPNETCORE_RUNTIME_VERSIONS="${ASPNETCORERUNTIMEVERSIONS:-""}" set -e diff --git a/src/java/devcontainer-feature.json b/src/java/devcontainer-feature.json index f65a8f360..9ed72ad59 100644 --- a/src/java/devcontainer-feature.json +++ b/src/java/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "java", - "version": "1.4.0", + "version": "1.4.1", "name": "Java (via SDKMAN!)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/java", "description": "Installs Java, SDKMAN! (if not installed), and needed dependencies.", diff --git a/src/java/install.sh b/src/java/install.sh index 37cb9e52b..30b6cb10c 100644 --- a/src/java/install.sh +++ b/src/java/install.sh @@ -18,7 +18,7 @@ INSTALL_ANT="${INSTALLANT:-"false"}" ANT_VERSION="${ANTVERSION:-"latest"}" INSTALL_GROOVY="${INSTALLGROOVY:-"false"}" GROOVY_VERSION="${GROOVYVERSION:-"latest"}" -JDK_DISTRO="${JDKDISTRO}" +JDK_DISTRO="${JDKDISTRO:-"ms"}" export SDKMAN_DIR="${SDKMAN_DIR:-"/usr/local/sdkman"}" USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index 0c15b6888..5a831f7e3 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.3.0", + "version": "1.3.1", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/powershell/install.sh b/src/powershell/install.sh index dadbc306f..2c052d7cd 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -13,7 +13,7 @@ set -e rm -rf /var/lib/apt/lists/* POWERSHELL_VERSION=${VERSION:-"latest"} -POWERSHELL_MODULES="${MODULES}" +POWERSHELL_MODULES="${MODULES:-""}" POWERSHELL_PROFILE_URL="${PROFILE_URL}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" From a8dc2dbb5909de35662067bb9428a7aafc10d3fe Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Mon, 26 Feb 2024 09:28:00 -0800 Subject: [PATCH 005/247] Docker: Unpin "mobyBuildxVersion" (#877) --- .../devcontainer-feature.json | 6 +++--- src/docker-in-docker/install.sh | 14 +++++++------ .../devcontainer-feature.json | 6 +++--- src/docker-outside-of-docker/install.sh | 2 +- .../docker_specific_moby_buildx.sh | 20 +++++++++++++++++++ test/docker-in-docker/scenarios.json | 8 ++++++++ test/docker-in-docker/test.sh | 1 + .../docker_specific_moby_buildx.sh | 19 ++++++++++++++++++ test/docker-outside-of-docker/scenarios.json | 8 ++++++++ test/docker-outside-of-docker/test.sh | 1 + 10 files changed, 72 insertions(+), 13 deletions(-) create mode 100755 test/docker-in-docker/docker_specific_moby_buildx.sh create mode 100755 test/docker-outside-of-docker/docker_specific_moby_buildx.sh diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index 5c62a057d..d74f205a1 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.9.1", + "version": "2.9.2", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", @@ -22,8 +22,8 @@ }, "mobyBuildxVersion": { "type": "string", - "default": "0.12.0", - "description": "Install a specific version of moby-buildx when using Moby. (2024-02-09: Microsoft's Package Manifest has mismatching filesize and SHA for 0.12.1; default is last known good version)" + "default": "latest", + "description": "Install a specific version of moby-buildx when using Moby" }, "dockerDashComposeVersion": { "type": "string", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index 889d1dbb2..9f0e18cee 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -10,7 +10,7 @@ DOCKER_VERSION="${VERSION:-"latest"}" # The Docker/Moby Engine + CLI should match in version USE_MOBY="${MOBY:-"true"}" -MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"0.12.0"}" +MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"latest"}" DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v1"}" # v1 or v2 or none AZURE_DNS_AUTO_DETECTION="${AZUREDNSAUTODETECTION:-"true"}" DOCKER_DEFAULT_ADDRESS_POOL="${DOCKERDEFAULTADDRESSPOOL:-""}" @@ -228,11 +228,13 @@ else # Install engine set +e # Handle error gracefully apt-get -y install --no-install-recommends moby-cli${cli_version_suffix} moby-buildx${buildx_version_suffix} moby-engine${engine_version_suffix} - if [ $? -ne 0 ]; then - err "Packages for moby not available in OS ${ID} ${VERSION_CODENAME} (${architecture}). To resolve, either: (1) set feature option '\"moby\": false' , or (2) choose a compatible OS version (eg: 'ubuntu-20.04')." - exit 1 - fi - set -e + exit_code=$? + set -e + + if [ ${exit_code} -ne 0 ]; then + err "Packages for moby not available in OS ${ID} ${VERSION_CODENAME} (${architecture}). To resolve, either: (1) set feature option '\"moby\": false' , or (2) choose a compatible OS version (eg: 'ubuntu-20.04')." + exit 1 + fi # Install compose apt-get -y install --no-install-recommends moby-compose || err "Package moby-compose (Docker Compose v2) not available for OS ${ID} ${VERSION_CODENAME} (${architecture}). Skipping." diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index a17757c06..d4c1447ba 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.4.1", + "version": "1.4.2", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", @@ -22,8 +22,8 @@ }, "mobyBuildxVersion": { "type": "string", - "default": "0.12.0", - "description": "Install a specific version of moby-buildx when using Moby. (2024-02-09: Microsoft's Package Manifest has mismatching filesize and SHA for 0.12.1; default is last known good version)" + "default": "latest", + "description": "Install a specific version of moby-buildx when using Moby" }, "dockerDashComposeVersion": { "type": "string", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index da936235c..65424740e 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -9,7 +9,7 @@ DOCKER_VERSION="${VERSION:-"latest"}" USE_MOBY="${MOBY:-"true"}" -MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"0.12.0"}" +MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"latest"}" DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v1"}" # v1 or v2 or none ENABLE_NONROOT_DOCKER="${ENABLE_NONROOT_DOCKER:-"true"}" diff --git a/test/docker-in-docker/docker_specific_moby_buildx.sh b/test/docker-in-docker/docker_specific_moby_buildx.sh new file mode 100755 index 000000000..1ca2b20bd --- /dev/null +++ b/test/docker-in-docker/docker_specific_moby_buildx.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib +# Definition specific tests +check "moby-buildx" bash -c "dpkg-query -W moby-buildx | grep -E '0.12.0'" + +check "docker-buildx" bash -c "docker buildx version" +check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" + +check "docker-buildx" docker buildx version +check "docker-build" docker build ./ + +check "installs docker-compose v1 install" bash -c "type docker-compose" +check "installs compose-switch" bash -c "[[ -f /usr/local/bin/compose-switch ]]" + +# Report result +reportResults diff --git a/test/docker-in-docker/scenarios.json b/test/docker-in-docker/scenarios.json index df27d6c28..ccf57b188 100644 --- a/test/docker-in-docker/scenarios.json +++ b/test/docker-in-docker/scenarios.json @@ -98,6 +98,14 @@ } } }, + "docker_specific_moby_buildx": { + "image": "ubuntu:focal", + "features": { + "docker-in-docker": { + "mobyBuildxVersion": "0.12.0" + } + } + }, // DO NOT REMOVE: This scenario is used by the docker-in-docker-stress-test workflow "docker_with_on_create_command": { "image": "mcr.microsoft.com/devcontainers/base:debian", diff --git a/test/docker-in-docker/test.sh b/test/docker-in-docker/test.sh index e86a841c2..10a7232b6 100755 --- a/test/docker-in-docker/test.sh +++ b/test/docker-in-docker/test.sh @@ -12,6 +12,7 @@ check "docker-ps" bash -c "docker ps" check "log-exists" bash -c "ls /tmp/dockerd.log" check "log-for-completion" bash -c "cat /tmp/dockerd.log | grep 'Daemon has completed initialization'" check "log-contents" bash -c "cat /tmp/dockerd.log | grep 'API listen on /var/run/docker.sock'" +check "moby-buildx" bash -c "dpkg-query -W moby-buildx" # Report result reportResults \ No newline at end of file diff --git a/test/docker-outside-of-docker/docker_specific_moby_buildx.sh b/test/docker-outside-of-docker/docker_specific_moby_buildx.sh new file mode 100755 index 000000000..929fa6080 --- /dev/null +++ b/test/docker-outside-of-docker/docker_specific_moby_buildx.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib +# Definition specific tests +check "moby-buildx" bash -c "dpkg-query -W moby-buildx | grep -E '0.12.0'" + +check "docker-buildx" bash -c "docker buildx version" +check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" + +check "docker-buildx" docker buildx version +check "docker-build" docker build ./ + +check "installs docker-compose v1 install" bash -c "type docker-compose" + +# Report result +reportResults diff --git a/test/docker-outside-of-docker/scenarios.json b/test/docker-outside-of-docker/scenarios.json index 3b82c6cc5..61f1ab402 100644 --- a/test/docker-outside-of-docker/scenarios.json +++ b/test/docker-outside-of-docker/scenarios.json @@ -124,5 +124,13 @@ } }, "remoteUser": "node" + }, + "docker_specific_moby_buildx": { + "image": "ubuntu:focal", + "features": { + "docker-outside-of-docker": { + "mobyBuildxVersion": "0.12.0" + } + } } } diff --git a/test/docker-outside-of-docker/test.sh b/test/docker-outside-of-docker/test.sh index fe9098e10..5206f5977 100644 --- a/test/docker-outside-of-docker/test.sh +++ b/test/docker-outside-of-docker/test.sh @@ -10,6 +10,7 @@ check "docker compose" bash -c "docker compose version" check "docker-compose" bash -c "docker-compose --version" check "docker-ps" bash -c "docker ps >/dev/null" +check "moby-buildx" bash -c "dpkg-query -W moby-buildx" # Report result reportResults \ No newline at end of file From 979c12bd966bb4ded8e9ac74a2b656f3eeaa77d0 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Mon, 26 Feb 2024 09:30:24 -0800 Subject: [PATCH 006/247] Automated documentation update (#881) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/docker-in-docker/README.md | 2 +- src/docker-outside-of-docker/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/docker-in-docker/README.md b/src/docker-in-docker/README.md index 41004b7e1..2ab700118 100644 --- a/src/docker-in-docker/README.md +++ b/src/docker-in-docker/README.md @@ -17,7 +17,7 @@ Create child containers *inside* a container, independent from the host's docker |-----|-----|-----|-----| | version | Select or enter a Docker/Moby Engine version. (Availability can vary by OS version.) | string | latest | | moby | Install OSS Moby build instead of Docker CE | boolean | true | -| mobyBuildxVersion | Install a specific version of moby-buildx when using Moby. (2024-02-09: Microsoft's Package Manifest has mismatching filesize and SHA for 0.12.1; default is last known good version) | string | 0.12.0 | +| mobyBuildxVersion | Install a specific version of moby-buildx when using Moby | string | latest | | dockerDashComposeVersion | Default version of Docker Compose (v1 or v2 or none) | string | v1 | | azureDnsAutoDetection | Allow automatically setting the dockerd DNS server when the installation script detects it is running in Azure | boolean | true | | dockerDefaultAddressPool | Define default address pools for Docker networks. e.g. base=192.168.0.0/16,size=24 | string | - | diff --git a/src/docker-outside-of-docker/README.md b/src/docker-outside-of-docker/README.md index 7e352af05..18fe79573 100644 --- a/src/docker-outside-of-docker/README.md +++ b/src/docker-outside-of-docker/README.md @@ -19,7 +19,7 @@ Re-use the host docker socket, adding the Docker CLI to a container. Feature inv |-----|-----|-----|-----| | version | Select or enter a Docker/Moby CLI version. (Availability can vary by OS version.) | string | latest | | moby | Install OSS Moby build instead of Docker CE | boolean | true | -| mobyBuildxVersion | Install a specific version of moby-buildx when using Moby. (2024-02-09: Microsoft's Package Manifest has mismatching filesize and SHA for 0.12.1; default is last known good version) | string | 0.12.0 | +| mobyBuildxVersion | Install a specific version of moby-buildx when using Moby | string | latest | | dockerDashComposeVersion | Compose version to use for docker-compose (v1 or v2 or none) | string | v2 | | installDockerBuildx | Install Docker Buildx | boolean | true | From 0604e45e6bcc6f4fa0fa6b4ea5ab58f47b2acba8 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 27 Feb 2024 09:20:42 -0800 Subject: [PATCH 007/247] Oryx: Fix build failures ; pin to .NET 8.0.101 (#882) * Oryx: Fix build failures ; pin to .NET 8.0.101 * fix build * fix "install_dotnet_and_oryx" --- .../update-dotnet-install-script.yml | 9 + src/oryx/devcontainer-feature.json | 2 +- src/oryx/install.sh | 43 +- src/oryx/scripts/vendor/README.md | 27 + src/oryx/scripts/vendor/dotnet-install.sh | 1844 +++++++++++++++++ 5 files changed, 1921 insertions(+), 4 deletions(-) create mode 100644 src/oryx/scripts/vendor/README.md create mode 100755 src/oryx/scripts/vendor/dotnet-install.sh diff --git a/.github/workflows/update-dotnet-install-script.yml b/.github/workflows/update-dotnet-install-script.yml index ebd175cff..85a9ea4d8 100644 --- a/.github/workflows/update-dotnet-install-script.yml +++ b/.github/workflows/update-dotnet-install-script.yml @@ -25,6 +25,9 @@ jobs: set -e echo "Start." + # Update dotnet-install for Oryx Feature as well + cp src/dotnet/scripts/vendor/dotnet-install.sh src/oryx/scripts/vendor/dotnet-install.sh + # Configure git and Push updates git config --global user.email github-actions@github.com git config --global user.name github-actions @@ -36,12 +39,18 @@ jobs: # Add / update and commit git add src/dotnet/scripts/vendor/dotnet-install.sh + git add src/dotnet/scripts/vendor/dotnet-install.sh + git commit -m 'Automated dotnet-install script update' || export NO_UPDATES=true # Bump version and push if [ "$NO_UPDATES" != "true" ] ; then echo "$(jq --indent 4 '.version = (.version | split(".") | map(tonumber) | .[2] += 1 | join("."))' src/dotnet/devcontainer-feature.json)" > src/dotnet/devcontainer-feature.json git add src/dotnet/devcontainer-feature.json + + echo "$(jq --indent 4 '.version = (.version | split(".") | map(tonumber) | .[2] += 1 | join("."))' src/oryx/devcontainer-feature.json)" > src/oryx/devcontainer-feature.json + git add src/oryx/devcontainer-feature.json + git commit -m 'Bump version' git push origin "$branch" gh api \ diff --git a/src/oryx/devcontainer-feature.json b/src/oryx/devcontainer-feature.json index aae6d43db..fe13e2fcb 100644 --- a/src/oryx/devcontainer-feature.json +++ b/src/oryx/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "oryx", - "version": "1.2.0", + "version": "1.3.0", "name": "Oryx", "description": "Installs the oryx CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/oryx", diff --git a/src/oryx/install.sh b/src/oryx/install.sh index a16890bef..f4c547fb5 100755 --- a/src/oryx/install.sh +++ b/src/oryx/install.sh @@ -70,6 +70,25 @@ check_packages() { fi } +install_dotnet_with_script() +{ + local version="$1" + CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") + DOTNET_INSTALL_SCRIPT="$CURRENT_DIR/scripts/vendor/dotnet-install.sh" + DOTNET_INSTALL_DIR='/usr/share/dotnet' + + check_packages icu-devtools + + "$DOTNET_INSTALL_SCRIPT" \ + --version "$version" \ + --install-dir "$DOTNET_INSTALL_DIR" \ + --no-path + + DOTNET_BINARY="dotnet" + export PATH="${PATH}:/usr/share/dotnet" + DOTNET_BINARY_INSTALLATION="/usr/share/dotnet/sdk/${version}" +} + install_dotnet_using_apt() { echo "Attempting to auto-install dotnet..." install_from_microsoft_feed=false @@ -86,6 +105,7 @@ install_dotnet_using_apt() { DOTNET_SKIP_FIRST_TIME_EXPERIENCE="true" apt-get install -yq $DOTNET_INSTALLATION_PACKAGE fi + DOTNET_BINARY="/usr/bin/dotnet" echo -e "Finished attempt to install dotnet. Sdks installed:\n" dotnet --list-sdks @@ -126,6 +146,7 @@ usermod -a -G oryx "${USERNAME}" # Required to decide if we want to clean up dotnet later. DOTNET_INSTALLATION_PACKAGE="" +DOTNET_BINARY_INSTALLATION="" DOTNET_BINARY="" if dotnet --version > /dev/null ; then @@ -133,18 +154,23 @@ if dotnet --version > /dev/null ; then fi MAJOR_VERSION_ID=$(echo $(dotnet --version) | cut -d . -f 1) +PATCH_VERSION_ID=$(echo $(dotnet --version) | cut -d . -f 3) # Oryx needs to be built with .NET 8 -if [[ "${DOTNET_BINARY}" = "" ]] || [[ $MAJOR_VERSION_ID != "8" ]] ; then +if [[ "${DOTNET_BINARY}" = "" ]] || [[ $MAJOR_VERSION_ID != "8" ]] || [[ $MAJOR_VERSION_ID = "8" && ${PATCH_VERSION_ID} -ge "101" ]] ; then echo "'dotnet 8' was not detected. Attempting to install .NET 8 to build oryx." - install_dotnet_using_apt + + # The oryx build fails with .Net 8.0.201, see https://github.com/devcontainers/images/issues/974 + # Pinning it to a working version until the upstream Oryx repo updates the dependency + # install_dotnet_using_apt + PINNED_SDK_VERSION="8.0.101" + install_dotnet_with_script ${PINNED_SDK_VERSION} if ! dotnet --version > /dev/null ; then echo "(!) Please install Dotnet before installing Oryx" exit 1 fi - DOTNET_BINARY="/usr/bin/dotnet" fi BUILD_SCRIPT_GENERATOR=/usr/local/buildscriptgen @@ -156,6 +182,11 @@ mkdir -p ${ORYX} git clone --depth=1 https://github.com/microsoft/Oryx $GIT_ORYX +if [[ "${DOTNET_BINARY_INSTALLATION}" != "" ]]; then + cd $GIT_ORYX + dotnet new globaljson --sdk-version ${PINNED_SDK_VERSION} +fi + SOLUTION_FILE_NAME="Oryx.sln" echo "Building solution '$SOLUTION_FILE_NAME'..." @@ -203,6 +234,12 @@ if [[ "${DOTNET_INSTALLATION_PACKAGE}" != "" ]]; then apt purge -yq $DOTNET_INSTALLATION_PACKAGE fi +if [[ "${DOTNET_BINARY_INSTALLATION}" != "" ]]; then + rm -f ${GIT_ORYX}/global.json + rm -rf ${DOTNET_BINARY_INSTALLATION} +fi + + # Clean up rm -rf /var/lib/apt/lists/* diff --git a/src/oryx/scripts/vendor/README.md b/src/oryx/scripts/vendor/README.md new file mode 100644 index 000000000..181b53781 --- /dev/null +++ b/src/oryx/scripts/vendor/README.md @@ -0,0 +1,27 @@ +### **IMPORTANT NOTE** + +Scripts in this directory are sourced externally and not maintained by the Dev Container spec maintainers. Do not make changes directly as they might be overwritten at any moment. + +## dotnet-install.sh + +`dotnet-install.sh` is a copy of . ([Script reference](https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script)) + +Quick options reminder for `dotnet-install.sh`: + +- `--version`: `"latest"` (default) or an exact version in the form A.B.C like `"6.0.413"` +- `--channel`: `"LTS"` (default), `"STS"`, a two-part version in the form A.B like `"6.0"` or three-part form A.B.Cxx like `"6.0.1xx"` +- `--quality`: `"daily"`, `"preview"` or `"GA"` +- The channel option is only used when version is 'latest' because an exact version overrides the channel option +- The quality option is only used when channel is 'A.B' or 'A.B.Cxx' because it can't be used with STS or LTS + +Examples + +``` +dotnet-install.sh [--version latest] [--channel LTS] +dotnet-install.sh [--version latest] --channel STS +dotnet-install.sh [--version latest] --channel 6.0 [--quality GA] +dotnet-install.sh [--version latest] --channel 6.0.4xx [--quality GA] +dotnet-install.sh [--version latest] --channel 8.0 --quality preview +dotnet-install.sh [--version latest] --channel 8.0 --quality daily +dotnet-install.sh --version 6.0.413 +``` \ No newline at end of file diff --git a/src/oryx/scripts/vendor/dotnet-install.sh b/src/oryx/scripts/vendor/dotnet-install.sh new file mode 100755 index 000000000..f6b08d1f8 --- /dev/null +++ b/src/oryx/scripts/vendor/dotnet-install.sh @@ -0,0 +1,1844 @@ +#!/usr/bin/env bash +# Copyright (c) .NET Foundation and contributors. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# + +# Stop script on NZEC +set -e +# Stop script if unbound variable found (use ${var:-} if intentional) +set -u +# By default cmd1 | cmd2 returns exit code of cmd2 regardless of cmd1 success +# This is causing it to fail +set -o pipefail + +# Use in the the functions: eval $invocation +invocation='say_verbose "Calling: ${yellow:-}${FUNCNAME[0]} ${green:-}$*${normal:-}"' + +# standard output may be used as a return value in the functions +# we need a way to write text on the screen in the functions so that +# it won't interfere with the return value. +# Exposing stream 3 as a pipe to standard output of the script itself +exec 3>&1 + +# Setup some colors to use. These need to work in fairly limited shells, like the Ubuntu Docker container where there are only 8 colors. +# See if stdout is a terminal +if [ -t 1 ] && command -v tput > /dev/null; then + # see if it supports colors + ncolors=$(tput colors || echo 0) + if [ -n "$ncolors" ] && [ $ncolors -ge 8 ]; then + bold="$(tput bold || echo)" + normal="$(tput sgr0 || echo)" + black="$(tput setaf 0 || echo)" + red="$(tput setaf 1 || echo)" + green="$(tput setaf 2 || echo)" + yellow="$(tput setaf 3 || echo)" + blue="$(tput setaf 4 || echo)" + magenta="$(tput setaf 5 || echo)" + cyan="$(tput setaf 6 || echo)" + white="$(tput setaf 7 || echo)" + fi +fi + +say_warning() { + printf "%b\n" "${yellow:-}dotnet_install: Warning: $1${normal:-}" >&3 +} + +say_err() { + printf "%b\n" "${red:-}dotnet_install: Error: $1${normal:-}" >&2 +} + +say() { + # using stream 3 (defined in the beginning) to not interfere with stdout of functions + # which may be used as return value + printf "%b\n" "${cyan:-}dotnet-install:${normal:-} $1" >&3 +} + +say_verbose() { + if [ "$verbose" = true ]; then + say "$1" + fi +} + +# This platform list is finite - if the SDK/Runtime has supported Linux distribution-specific assets, +# then and only then should the Linux distribution appear in this list. +# Adding a Linux distribution to this list does not imply distribution-specific support. +get_legacy_os_name_from_platform() { + eval $invocation + + platform="$1" + case "$platform" in + "centos.7") + echo "centos" + return 0 + ;; + "debian.8") + echo "debian" + return 0 + ;; + "debian.9") + echo "debian.9" + return 0 + ;; + "fedora.23") + echo "fedora.23" + return 0 + ;; + "fedora.24") + echo "fedora.24" + return 0 + ;; + "fedora.27") + echo "fedora.27" + return 0 + ;; + "fedora.28") + echo "fedora.28" + return 0 + ;; + "opensuse.13.2") + echo "opensuse.13.2" + return 0 + ;; + "opensuse.42.1") + echo "opensuse.42.1" + return 0 + ;; + "opensuse.42.3") + echo "opensuse.42.3" + return 0 + ;; + "rhel.7"*) + echo "rhel" + return 0 + ;; + "ubuntu.14.04") + echo "ubuntu" + return 0 + ;; + "ubuntu.16.04") + echo "ubuntu.16.04" + return 0 + ;; + "ubuntu.16.10") + echo "ubuntu.16.10" + return 0 + ;; + "ubuntu.18.04") + echo "ubuntu.18.04" + return 0 + ;; + "alpine.3.4.3") + echo "alpine" + return 0 + ;; + esac + return 1 +} + +get_legacy_os_name() { + eval $invocation + + local uname=$(uname) + if [ "$uname" = "Darwin" ]; then + echo "osx" + return 0 + elif [ -n "$runtime_id" ]; then + echo $(get_legacy_os_name_from_platform "${runtime_id%-*}" || echo "${runtime_id%-*}") + return 0 + else + if [ -e /etc/os-release ]; then + . /etc/os-release + os=$(get_legacy_os_name_from_platform "$ID${VERSION_ID:+.${VERSION_ID}}" || echo "") + if [ -n "$os" ]; then + echo "$os" + return 0 + fi + fi + fi + + say_verbose "Distribution specific OS name and version could not be detected: UName = $uname" + return 1 +} + +get_linux_platform_name() { + eval $invocation + + if [ -n "$runtime_id" ]; then + echo "${runtime_id%-*}" + return 0 + else + if [ -e /etc/os-release ]; then + . /etc/os-release + echo "$ID${VERSION_ID:+.${VERSION_ID}}" + return 0 + elif [ -e /etc/redhat-release ]; then + local redhatRelease=$(&1 || true) | grep -q musl +} + +get_current_os_name() { + eval $invocation + + local uname=$(uname) + if [ "$uname" = "Darwin" ]; then + echo "osx" + return 0 + elif [ "$uname" = "FreeBSD" ]; then + echo "freebsd" + return 0 + elif [ "$uname" = "Linux" ]; then + local linux_platform_name="" + linux_platform_name="$(get_linux_platform_name)" || true + + if [ "$linux_platform_name" = "rhel.6" ]; then + echo $linux_platform_name + return 0 + elif is_musl_based_distro; then + echo "linux-musl" + return 0 + elif [ "$linux_platform_name" = "linux-musl" ]; then + echo "linux-musl" + return 0 + else + echo "linux" + return 0 + fi + fi + + say_err "OS name could not be detected: UName = $uname" + return 1 +} + +machine_has() { + eval $invocation + + command -v "$1" > /dev/null 2>&1 + return $? +} + +check_min_reqs() { + local hasMinimum=false + if machine_has "curl"; then + hasMinimum=true + elif machine_has "wget"; then + hasMinimum=true + fi + + if [ "$hasMinimum" = "false" ]; then + say_err "curl (recommended) or wget are required to download dotnet. Install missing prerequisite to proceed." + return 1 + fi + return 0 +} + +# args: +# input - $1 +to_lowercase() { + #eval $invocation + + echo "$1" | tr '[:upper:]' '[:lower:]' + return 0 +} + +# args: +# input - $1 +remove_trailing_slash() { + #eval $invocation + + local input="${1:-}" + echo "${input%/}" + return 0 +} + +# args: +# input - $1 +remove_beginning_slash() { + #eval $invocation + + local input="${1:-}" + echo "${input#/}" + return 0 +} + +# args: +# root_path - $1 +# child_path - $2 - this parameter can be empty +combine_paths() { + eval $invocation + + # TODO: Consider making it work with any number of paths. For now: + if [ ! -z "${3:-}" ]; then + say_err "combine_paths: Function takes two parameters." + return 1 + fi + + local root_path="$(remove_trailing_slash "$1")" + local child_path="$(remove_beginning_slash "${2:-}")" + say_verbose "combine_paths: root_path=$root_path" + say_verbose "combine_paths: child_path=$child_path" + echo "$root_path/$child_path" + return 0 +} + +get_machine_architecture() { + eval $invocation + + if command -v uname > /dev/null; then + CPUName=$(uname -m) + case $CPUName in + armv*l) + echo "arm" + return 0 + ;; + aarch64|arm64) + if [ "$(getconf LONG_BIT)" -lt 64 ]; then + # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS) + echo "arm" + return 0 + fi + echo "arm64" + return 0 + ;; + s390x) + echo "s390x" + return 0 + ;; + ppc64le) + echo "ppc64le" + return 0 + ;; + loongarch64) + echo "loongarch64" + return 0 + ;; + esac + fi + + # Always default to 'x64' + echo "x64" + return 0 +} + +# args: +# architecture - $1 +get_normalized_architecture_from_architecture() { + eval $invocation + + local architecture="$(to_lowercase "$1")" + + if [[ $architecture == \ ]]; then + echo "$(get_machine_architecture)" + return 0 + fi + + case "$architecture" in + amd64|x64) + echo "x64" + return 0 + ;; + arm) + echo "arm" + return 0 + ;; + arm64) + echo "arm64" + return 0 + ;; + s390x) + echo "s390x" + return 0 + ;; + ppc64le) + echo "ppc64le" + return 0 + ;; + loongarch64) + echo "loongarch64" + return 0 + ;; + esac + + say_err "Architecture \`$architecture\` not supported. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues" + return 1 +} + +# args: +# version - $1 +# channel - $2 +# architecture - $3 +get_normalized_architecture_for_specific_sdk_version() { + eval $invocation + + local is_version_support_arm64="$(is_arm64_supported "$1")" + local is_channel_support_arm64="$(is_arm64_supported "$2")" + local architecture="$3"; + local osname="$(get_current_os_name)" + + if [ "$osname" == "osx" ] && [ "$architecture" == "arm64" ] && { [ "$is_version_support_arm64" = false ] || [ "$is_channel_support_arm64" = false ]; }; then + #check if rosetta is installed + if [ "$(/usr/bin/pgrep oahd >/dev/null 2>&1;echo $?)" -eq 0 ]; then + say_verbose "Changing user architecture from '$architecture' to 'x64' because .NET SDKs prior to version 6.0 do not support arm64." + echo "x64" + return 0; + else + say_err "Architecture \`$architecture\` is not supported for .NET SDK version \`$version\`. Please install Rosetta to allow emulation of the \`$architecture\` .NET SDK on this platform" + return 1 + fi + fi + + echo "$architecture" + return 0 +} + +# args: +# version or channel - $1 +is_arm64_supported() { + #any channel or version that starts with the specified versions + case "$1" in + ( "1"* | "2"* | "3"* | "4"* | "5"*) + echo false + return 0 + esac + + echo true + return 0 +} + +# args: +# user_defined_os - $1 +get_normalized_os() { + eval $invocation + + local osname="$(to_lowercase "$1")" + if [ ! -z "$osname" ]; then + case "$osname" in + osx | freebsd | rhel.6 | linux-musl | linux) + echo "$osname" + return 0 + ;; + macos) + osname='osx' + echo "$osname" + return 0 + ;; + *) + say_err "'$user_defined_os' is not a supported value for --os option, supported values are: osx, macos, linux, linux-musl, freebsd, rhel.6. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues." + return 1 + ;; + esac + else + osname="$(get_current_os_name)" || return 1 + fi + echo "$osname" + return 0 +} + +# args: +# quality - $1 +get_normalized_quality() { + eval $invocation + + local quality="$(to_lowercase "$1")" + if [ ! -z "$quality" ]; then + case "$quality" in + daily | signed | validated | preview) + echo "$quality" + return 0 + ;; + ga) + #ga quality is available without specifying quality, so normalizing it to empty + return 0 + ;; + *) + say_err "'$quality' is not a supported value for --quality option. Supported values are: daily, signed, validated, preview, ga. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues." + return 1 + ;; + esac + fi + return 0 +} + +# args: +# channel - $1 +get_normalized_channel() { + eval $invocation + + local channel="$(to_lowercase "$1")" + + if [[ $channel == current ]]; then + say_warning 'Value "Current" is deprecated for -Channel option. Use "STS" instead.' + fi + + if [[ $channel == release/* ]]; then + say_warning 'Using branch name with -Channel option is no longer supported with newer releases. Use -Quality option with a channel in X.Y format instead.'; + fi + + if [ ! -z "$channel" ]; then + case "$channel" in + lts) + echo "LTS" + return 0 + ;; + sts) + echo "STS" + return 0 + ;; + current) + echo "STS" + return 0 + ;; + *) + echo "$channel" + return 0 + ;; + esac + fi + + return 0 +} + +# args: +# runtime - $1 +get_normalized_product() { + eval $invocation + + local product="" + local runtime="$(to_lowercase "$1")" + if [[ "$runtime" == "dotnet" ]]; then + product="dotnet-runtime" + elif [[ "$runtime" == "aspnetcore" ]]; then + product="aspnetcore-runtime" + elif [ -z "$runtime" ]; then + product="dotnet-sdk" + fi + echo "$product" + return 0 +} + +# The version text returned from the feeds is a 1-line or 2-line string: +# For the SDK and the dotnet runtime (2 lines): +# Line 1: # commit_hash +# Line 2: # 4-part version +# For the aspnetcore runtime (1 line): +# Line 1: # 4-part version + +# args: +# version_text - stdin +get_version_from_latestversion_file_content() { + eval $invocation + + cat | tail -n 1 | sed 's/\r$//' + return 0 +} + +# args: +# install_root - $1 +# relative_path_to_package - $2 +# specific_version - $3 +is_dotnet_package_installed() { + eval $invocation + + local install_root="$1" + local relative_path_to_package="$2" + local specific_version="${3//[$'\t\r\n']}" + + local dotnet_package_path="$(combine_paths "$(combine_paths "$install_root" "$relative_path_to_package")" "$specific_version")" + say_verbose "is_dotnet_package_installed: dotnet_package_path=$dotnet_package_path" + + if [ -d "$dotnet_package_path" ]; then + return 0 + else + return 1 + fi +} + +# args: +# downloaded file - $1 +# remote_file_size - $2 +validate_remote_local_file_sizes() +{ + eval $invocation + + local downloaded_file="$1" + local remote_file_size="$2" + local file_size='' + + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + file_size="$(stat -c '%s' "$downloaded_file")" + elif [[ "$OSTYPE" == "darwin"* ]]; then + # hardcode in order to avoid conflicts with GNU stat + file_size="$(/usr/bin/stat -f '%z' "$downloaded_file")" + fi + + if [ -n "$file_size" ]; then + say "Downloaded file size is $file_size bytes." + + if [ -n "$remote_file_size" ] && [ -n "$file_size" ]; then + if [ "$remote_file_size" -ne "$file_size" ]; then + say "The remote and local file sizes are not equal. The remote file size is $remote_file_size bytes and the local size is $file_size bytes. The local package may be corrupted." + else + say "The remote and local file sizes are equal." + fi + fi + + else + say "Either downloaded or local package size can not be measured. One of them may be corrupted." + fi +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +get_version_from_latestversion_file() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + + local version_file_url=null + if [[ "$runtime" == "dotnet" ]]; then + version_file_url="$azure_feed/Runtime/$channel/latest.version" + elif [[ "$runtime" == "aspnetcore" ]]; then + version_file_url="$azure_feed/aspnetcore/Runtime/$channel/latest.version" + elif [ -z "$runtime" ]; then + version_file_url="$azure_feed/Sdk/$channel/latest.version" + else + say_err "Invalid value for \$runtime" + return 1 + fi + say_verbose "get_version_from_latestversion_file: latest url: $version_file_url" + + download "$version_file_url" || return $? + return 0 +} + +# args: +# json_file - $1 +parse_globaljson_file_for_version() { + eval $invocation + + local json_file="$1" + if [ ! -f "$json_file" ]; then + say_err "Unable to find \`$json_file\`" + return 1 + fi + + sdk_section=$(cat $json_file | tr -d "\r" | awk '/"sdk"/,/}/') + if [ -z "$sdk_section" ]; then + say_err "Unable to parse the SDK node in \`$json_file\`" + return 1 + fi + + sdk_list=$(echo $sdk_section | awk -F"[{}]" '{print $2}') + sdk_list=${sdk_list//[\" ]/} + sdk_list=${sdk_list//,/$'\n'} + + local version_info="" + while read -r line; do + IFS=: + while read -r key value; do + if [[ "$key" == "version" ]]; then + version_info=$value + fi + done <<< "$line" + done <<< "$sdk_list" + if [ -z "$version_info" ]; then + say_err "Unable to find the SDK:version node in \`$json_file\`" + return 1 + fi + + unset IFS; + echo "$version_info" + return 0 +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +# version - $4 +# json_file - $5 +get_specific_version_from_version() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + local version="$(to_lowercase "$4")" + local json_file="$5" + + if [ -z "$json_file" ]; then + if [[ "$version" == "latest" ]]; then + local version_info + version_info="$(get_version_from_latestversion_file "$azure_feed" "$channel" "$normalized_architecture" false)" || return 1 + say_verbose "get_specific_version_from_version: version_info=$version_info" + echo "$version_info" | get_version_from_latestversion_file_content + return 0 + else + echo "$version" + return 0 + fi + else + local version_info + version_info="$(parse_globaljson_file_for_version "$json_file")" || return 1 + echo "$version_info" + return 0 + fi +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +# specific_version - $4 +# normalized_os - $5 +construct_download_link() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + local specific_version="${4//[$'\t\r\n']}" + local specific_product_version="$(get_specific_product_version "$1" "$4")" + local osname="$5" + + local download_link=null + if [[ "$runtime" == "dotnet" ]]; then + download_link="$azure_feed/Runtime/$specific_version/dotnet-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz" + elif [[ "$runtime" == "aspnetcore" ]]; then + download_link="$azure_feed/aspnetcore/Runtime/$specific_version/aspnetcore-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz" + elif [ -z "$runtime" ]; then + download_link="$azure_feed/Sdk/$specific_version/dotnet-sdk-$specific_product_version-$osname-$normalized_architecture.tar.gz" + else + return 1 + fi + + echo "$download_link" + return 0 +} + +# args: +# azure_feed - $1 +# specific_version - $2 +# download link - $3 (optional) +get_specific_product_version() { + # If we find a 'productVersion.txt' at the root of any folder, we'll use its contents + # to resolve the version of what's in the folder, superseding the specified version. + # if 'productVersion.txt' is missing but download link is already available, product version will be taken from download link + eval $invocation + + local azure_feed="$1" + local specific_version="${2//[$'\t\r\n']}" + local package_download_link="" + if [ $# -gt 2 ]; then + local package_download_link="$3" + fi + local specific_product_version=null + + # Try to get the version number, using the productVersion.txt file located next to the installer file. + local download_links=($(get_specific_product_version_url "$azure_feed" "$specific_version" true "$package_download_link") + $(get_specific_product_version_url "$azure_feed" "$specific_version" false "$package_download_link")) + + for download_link in "${download_links[@]}" + do + say_verbose "Checking for the existence of $download_link" + + if machine_has "curl" + then + if ! specific_product_version=$(curl -s --fail "${download_link}${feed_credential}" 2>&1); then + continue + else + echo "${specific_product_version//[$'\t\r\n']}" + return 0 + fi + + elif machine_has "wget" + then + specific_product_version=$(wget -qO- "${download_link}${feed_credential}" 2>&1) + if [ $? = 0 ]; then + echo "${specific_product_version//[$'\t\r\n']}" + return 0 + fi + fi + done + + # Getting the version number with productVersion.txt has failed. Try parsing the download link for a version number. + say_verbose "Failed to get the version using productVersion.txt file. Download link will be parsed instead." + specific_product_version="$(get_product_specific_version_from_download_link "$package_download_link" "$specific_version")" + echo "${specific_product_version//[$'\t\r\n']}" + return 0 +} + +# args: +# azure_feed - $1 +# specific_version - $2 +# is_flattened - $3 +# download link - $4 (optional) +get_specific_product_version_url() { + eval $invocation + + local azure_feed="$1" + local specific_version="$2" + local is_flattened="$3" + local package_download_link="" + if [ $# -gt 3 ]; then + local package_download_link="$4" + fi + + local pvFileName="productVersion.txt" + if [ "$is_flattened" = true ]; then + if [ -z "$runtime" ]; then + pvFileName="sdk-productVersion.txt" + elif [[ "$runtime" == "dotnet" ]]; then + pvFileName="runtime-productVersion.txt" + else + pvFileName="$runtime-productVersion.txt" + fi + fi + + local download_link=null + + if [ -z "$package_download_link" ]; then + if [[ "$runtime" == "dotnet" ]]; then + download_link="$azure_feed/Runtime/$specific_version/${pvFileName}" + elif [[ "$runtime" == "aspnetcore" ]]; then + download_link="$azure_feed/aspnetcore/Runtime/$specific_version/${pvFileName}" + elif [ -z "$runtime" ]; then + download_link="$azure_feed/Sdk/$specific_version/${pvFileName}" + else + return 1 + fi + else + download_link="${package_download_link%/*}/${pvFileName}" + fi + + say_verbose "Constructed productVersion link: $download_link" + echo "$download_link" + return 0 +} + +# args: +# download link - $1 +# specific version - $2 +get_product_specific_version_from_download_link() +{ + eval $invocation + + local download_link="$1" + local specific_version="$2" + local specific_product_version="" + + if [ -z "$download_link" ]; then + echo "$specific_version" + return 0 + fi + + #get filename + filename="${download_link##*/}" + + #product specific version follows the product name + #for filename 'dotnet-sdk-3.1.404-linux-x64.tar.gz': the product version is 3.1.404 + IFS='-' + read -ra filename_elems <<< "$filename" + count=${#filename_elems[@]} + if [[ "$count" -gt 2 ]]; then + specific_product_version="${filename_elems[2]}" + else + specific_product_version=$specific_version + fi + unset IFS; + echo "$specific_product_version" + return 0 +} + +# args: +# azure_feed - $1 +# channel - $2 +# normalized_architecture - $3 +# specific_version - $4 +construct_legacy_download_link() { + eval $invocation + + local azure_feed="$1" + local channel="$2" + local normalized_architecture="$3" + local specific_version="${4//[$'\t\r\n']}" + + local distro_specific_osname + distro_specific_osname="$(get_legacy_os_name)" || return 1 + + local legacy_download_link=null + if [[ "$runtime" == "dotnet" ]]; then + legacy_download_link="$azure_feed/Runtime/$specific_version/dotnet-$distro_specific_osname-$normalized_architecture.$specific_version.tar.gz" + elif [ -z "$runtime" ]; then + legacy_download_link="$azure_feed/Sdk/$specific_version/dotnet-dev-$distro_specific_osname-$normalized_architecture.$specific_version.tar.gz" + else + return 1 + fi + + echo "$legacy_download_link" + return 0 +} + +get_user_install_path() { + eval $invocation + + if [ ! -z "${DOTNET_INSTALL_DIR:-}" ]; then + echo "$DOTNET_INSTALL_DIR" + else + echo "$HOME/.dotnet" + fi + return 0 +} + +# args: +# install_dir - $1 +resolve_installation_path() { + eval $invocation + + local install_dir=$1 + if [ "$install_dir" = "" ]; then + local user_install_path="$(get_user_install_path)" + say_verbose "resolve_installation_path: user_install_path=$user_install_path" + echo "$user_install_path" + return 0 + fi + + echo "$install_dir" + return 0 +} + +# args: +# relative_or_absolute_path - $1 +get_absolute_path() { + eval $invocation + + local relative_or_absolute_path=$1 + echo "$(cd "$(dirname "$1")" && pwd -P)/$(basename "$1")" + return 0 +} + +# args: +# input_files - stdin +# root_path - $1 +# out_path - $2 +# override - $3 +copy_files_or_dirs_from_list() { + eval $invocation + + local root_path="$(remove_trailing_slash "$1")" + local out_path="$(remove_trailing_slash "$2")" + local override="$3" + local osname="$(get_current_os_name)" + local override_switch=$( + if [ "$override" = false ]; then + if [ "$osname" = "linux-musl" ]; then + printf -- "-u"; + else + printf -- "-n"; + fi + fi) + + cat | uniq | while read -r file_path; do + local path="$(remove_beginning_slash "${file_path#$root_path}")" + local target="$out_path/$path" + if [ "$override" = true ] || (! ([ -d "$target" ] || [ -e "$target" ])); then + mkdir -p "$out_path/$(dirname "$path")" + if [ -d "$target" ]; then + rm -rf "$target" + fi + cp -R $override_switch "$root_path/$path" "$target" + fi + done +} + +# args: +# zip_uri - $1 +get_remote_file_size() { + local zip_uri="$1" + + if machine_has "curl"; then + file_size=$(curl -sI "$zip_uri" | grep -i content-length | awk '{ num = $2 + 0; print num }') + elif machine_has "wget"; then + file_size=$(wget --spider --server-response -O /dev/null "$zip_uri" 2>&1 | grep -i 'Content-Length:' | awk '{ num = $2 + 0; print num }') + else + say "Neither curl nor wget is available on this system." + return + fi + + if [ -n "$file_size" ]; then + say "Remote file $zip_uri size is $file_size bytes." + echo "$file_size" + else + say_verbose "Content-Length header was not extracted for $zip_uri." + echo "" + fi +} + +# args: +# zip_path - $1 +# out_path - $2 +# remote_file_size - $3 +extract_dotnet_package() { + eval $invocation + + local zip_path="$1" + local out_path="$2" + local remote_file_size="$3" + + local temp_out_path="$(mktemp -d "$temporary_file_template")" + + local failed=false + tar -xzf "$zip_path" -C "$temp_out_path" > /dev/null || failed=true + + local folders_with_version_regex='^.*/[0-9]+\.[0-9]+[^/]+/' + find "$temp_out_path" -type f | grep -Eo "$folders_with_version_regex" | sort | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" false + find "$temp_out_path" -type f | grep -Ev "$folders_with_version_regex" | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" "$override_non_versioned_files" + + validate_remote_local_file_sizes "$zip_path" "$remote_file_size" + + rm -rf "$temp_out_path" + if [ -z ${keep_zip+x} ]; then + rm -f "$zip_path" && say_verbose "Temporary zip file $zip_path was removed" + fi + + if [ "$failed" = true ]; then + say_err "Extraction failed" + return 1 + fi + return 0 +} + +# args: +# remote_path - $1 +# disable_feed_credential - $2 +get_http_header() +{ + eval $invocation + local remote_path="$1" + local disable_feed_credential="$2" + + local failed=false + local response + if machine_has "curl"; then + get_http_header_curl $remote_path $disable_feed_credential || failed=true + elif machine_has "wget"; then + get_http_header_wget $remote_path $disable_feed_credential || failed=true + else + failed=true + fi + if [ "$failed" = true ]; then + say_verbose "Failed to get HTTP header: '$remote_path'." + return 1 + fi + return 0 +} + +# args: +# remote_path - $1 +# disable_feed_credential - $2 +get_http_header_curl() { + eval $invocation + local remote_path="$1" + local disable_feed_credential="$2" + + remote_path_with_credential="$remote_path" + if [ "$disable_feed_credential" = false ]; then + remote_path_with_credential+="$feed_credential" + fi + + curl_options="-I -sSL --retry 5 --retry-delay 2 --connect-timeout 15 " + curl $curl_options "$remote_path_with_credential" 2>&1 || return 1 + return 0 +} + +# args: +# remote_path - $1 +# disable_feed_credential - $2 +get_http_header_wget() { + eval $invocation + local remote_path="$1" + local disable_feed_credential="$2" + local wget_options="-q -S --spider --tries 5 " + + local wget_options_extra='' + + # Test for options that aren't supported on all wget implementations. + if [[ $(wget -h 2>&1 | grep -E 'waitretry|connect-timeout') ]]; then + wget_options_extra="--waitretry 2 --connect-timeout 15 " + else + say "wget extra options are unavailable for this environment" + fi + + remote_path_with_credential="$remote_path" + if [ "$disable_feed_credential" = false ]; then + remote_path_with_credential+="$feed_credential" + fi + + wget $wget_options $wget_options_extra "$remote_path_with_credential" 2>&1 + + return $? +} + +# args: +# remote_path - $1 +# [out_path] - $2 - stdout if not provided +download() { + eval $invocation + + local remote_path="$1" + local out_path="${2:-}" + + if [[ "$remote_path" != "http"* ]]; then + cp "$remote_path" "$out_path" + return $? + fi + + local failed=false + local attempts=0 + while [ $attempts -lt 3 ]; do + attempts=$((attempts+1)) + failed=false + if machine_has "curl"; then + downloadcurl "$remote_path" "$out_path" || failed=true + elif machine_has "wget"; then + downloadwget "$remote_path" "$out_path" || failed=true + else + say_err "Missing dependency: neither curl nor wget was found." + exit 1 + fi + + if [ "$failed" = false ] || [ $attempts -ge 3 ] || { [ ! -z $http_code ] && [ $http_code = "404" ]; }; then + break + fi + + say "Download attempt #$attempts has failed: $http_code $download_error_msg" + say "Attempt #$((attempts+1)) will start in $((attempts*10)) seconds." + sleep $((attempts*10)) + done + + if [ "$failed" = true ]; then + say_verbose "Download failed: $remote_path" + return 1 + fi + return 0 +} + +# Updates global variables $http_code and $download_error_msg +downloadcurl() { + eval $invocation + unset http_code + unset download_error_msg + local remote_path="$1" + local out_path="${2:-}" + # Append feed_credential as late as possible before calling curl to avoid logging feed_credential + # Avoid passing URI with credentials to functions: note, most of them echoing parameters of invocation in verbose output. + local remote_path_with_credential="${remote_path}${feed_credential}" + local curl_options="--retry 20 --retry-delay 2 --connect-timeout 15 -sSL -f --create-dirs " + local curl_exit_code=0; + if [ -z "$out_path" ]; then + curl $curl_options "$remote_path_with_credential" 2>&1 + curl_exit_code=$? + else + curl $curl_options -o "$out_path" "$remote_path_with_credential" 2>&1 + curl_exit_code=$? + fi + + if [ $curl_exit_code -gt 0 ]; then + download_error_msg="Unable to download $remote_path." + # Check for curl timeout codes + if [[ $curl_exit_code == 7 || $curl_exit_code == 28 ]]; then + download_error_msg+=" Failed to reach the server: connection timeout." + else + local disable_feed_credential=false + local response=$(get_http_header_curl $remote_path $disable_feed_credential) + http_code=$( echo "$response" | awk '/^HTTP/{print $2}' | tail -1 ) + if [[ ! -z $http_code && $http_code != 2* ]]; then + download_error_msg+=" Returned HTTP status code: $http_code." + fi + fi + say_verbose "$download_error_msg" + return 1 + fi + return 0 +} + + +# Updates global variables $http_code and $download_error_msg +downloadwget() { + eval $invocation + unset http_code + unset download_error_msg + local remote_path="$1" + local out_path="${2:-}" + # Append feed_credential as late as possible before calling wget to avoid logging feed_credential + local remote_path_with_credential="${remote_path}${feed_credential}" + local wget_options="--tries 20 " + + local wget_options_extra='' + local wget_result='' + + # Test for options that aren't supported on all wget implementations. + if [[ $(wget -h 2>&1 | grep -E 'waitretry|connect-timeout') ]]; then + wget_options_extra="--waitretry 2 --connect-timeout 15 " + else + say "wget extra options are unavailable for this environment" + fi + + if [ -z "$out_path" ]; then + wget -q $wget_options $wget_options_extra -O - "$remote_path_with_credential" 2>&1 + wget_result=$? + else + wget $wget_options $wget_options_extra -O "$out_path" "$remote_path_with_credential" 2>&1 + wget_result=$? + fi + + if [[ $wget_result != 0 ]]; then + local disable_feed_credential=false + local response=$(get_http_header_wget $remote_path $disable_feed_credential) + http_code=$( echo "$response" | awk '/^ HTTP/{print $2}' | tail -1 ) + download_error_msg="Unable to download $remote_path." + if [[ ! -z $http_code && $http_code != 2* ]]; then + download_error_msg+=" Returned HTTP status code: $http_code." + # wget exit code 4 stands for network-issue + elif [[ $wget_result == 4 ]]; then + download_error_msg+=" Failed to reach the server: connection timeout." + fi + say_verbose "$download_error_msg" + return 1 + fi + + return 0 +} + +get_download_link_from_aka_ms() { + eval $invocation + + #quality is not supported for LTS or STS channel + #STS maps to current + if [[ ! -z "$normalized_quality" && ("$normalized_channel" == "LTS" || "$normalized_channel" == "STS") ]]; then + normalized_quality="" + say_warning "Specifying quality for STS or LTS channel is not supported, the quality will be ignored." + fi + + say_verbose "Retrieving primary payload URL from aka.ms for channel: '$normalized_channel', quality: '$normalized_quality', product: '$normalized_product', os: '$normalized_os', architecture: '$normalized_architecture'." + + #construct aka.ms link + aka_ms_link="https://aka.ms/dotnet" + if [ "$internal" = true ]; then + aka_ms_link="$aka_ms_link/internal" + fi + aka_ms_link="$aka_ms_link/$normalized_channel" + if [[ ! -z "$normalized_quality" ]]; then + aka_ms_link="$aka_ms_link/$normalized_quality" + fi + aka_ms_link="$aka_ms_link/$normalized_product-$normalized_os-$normalized_architecture.tar.gz" + say_verbose "Constructed aka.ms link: '$aka_ms_link'." + + #get HTTP response + #do not pass credentials as a part of the $aka_ms_link and do not apply credentials in the get_http_header function + #otherwise the redirect link would have credentials as well + #it would result in applying credentials twice to the resulting link and thus breaking it, and in echoing credentials to the output as a part of redirect link + disable_feed_credential=true + response="$(get_http_header $aka_ms_link $disable_feed_credential)" + + say_verbose "Received response: $response" + # Get results of all the redirects. + http_codes=$( echo "$response" | awk '$1 ~ /^HTTP/ {print $2}' ) + # They all need to be 301, otherwise some links are broken (except for the last, which is not a redirect but 200 or 404). + broken_redirects=$( echo "$http_codes" | sed '$d' | grep -v '301' ) + + # All HTTP codes are 301 (Moved Permanently), the redirect link exists. + if [[ -z "$broken_redirects" ]]; then + aka_ms_download_link=$( echo "$response" | awk '$1 ~ /^Location/{print $2}' | tail -1 | tr -d '\r') + + if [[ -z "$aka_ms_download_link" ]]; then + say_verbose "The aka.ms link '$aka_ms_link' is not valid: failed to get redirect location." + return 1 + fi + + say_verbose "The redirect location retrieved: '$aka_ms_download_link'." + return 0 + else + say_verbose "The aka.ms link '$aka_ms_link' is not valid: received HTTP code: $(echo "$broken_redirects" | paste -sd "," -)." + return 1 + fi +} + +get_feeds_to_use() +{ + feeds=( + "https://dotnetcli.azureedge.net/dotnet" + "https://dotnetbuilds.azureedge.net/public" + ) + + if [[ -n "$azure_feed" ]]; then + feeds=("$azure_feed") + fi + + if [[ "$no_cdn" == "true" ]]; then + feeds=( + "https://dotnetcli.blob.core.windows.net/dotnet" + "https://dotnetbuilds.blob.core.windows.net/public" + ) + + if [[ -n "$uncached_feed" ]]; then + feeds=("$uncached_feed") + fi + fi +} + +# THIS FUNCTION MAY EXIT (if the determined version is already installed). +generate_download_links() { + + download_links=() + specific_versions=() + effective_versions=() + link_types=() + + # If generate_akams_links returns false, no fallback to old links. Just terminate. + # This function may also 'exit' (if the determined version is already installed). + generate_akams_links || return + + # Check other feeds only if we haven't been able to find an aka.ms link. + if [[ "${#download_links[@]}" -lt 1 ]]; then + for feed in ${feeds[@]} + do + # generate_regular_links may also 'exit' (if the determined version is already installed). + generate_regular_links $feed || return + done + fi + + if [[ "${#download_links[@]}" -eq 0 ]]; then + say_err "Failed to resolve the exact version number." + return 1 + fi + + say_verbose "Generated ${#download_links[@]} links." + for link_index in ${!download_links[@]} + do + say_verbose "Link $link_index: ${link_types[$link_index]}, ${effective_versions[$link_index]}, ${download_links[$link_index]}" + done +} + +# THIS FUNCTION MAY EXIT (if the determined version is already installed). +generate_akams_links() { + local valid_aka_ms_link=true; + + normalized_version="$(to_lowercase "$version")" + if [[ "$normalized_version" != "latest" ]] && [ -n "$normalized_quality" ]; then + say_err "Quality and Version options are not allowed to be specified simultaneously. See https://learn.microsoft.com/dotnet/core/tools/dotnet-install-script#options for details." + return 1 + fi + + if [[ -n "$json_file" || "$normalized_version" != "latest" ]]; then + # aka.ms links are not needed when exact version is specified via command or json file + return + fi + + get_download_link_from_aka_ms || valid_aka_ms_link=false + + if [[ "$valid_aka_ms_link" == true ]]; then + say_verbose "Retrieved primary payload URL from aka.ms link: '$aka_ms_download_link'." + say_verbose "Downloading using legacy url will not be attempted." + + download_link=$aka_ms_download_link + + #get version from the path + IFS='/' + read -ra pathElems <<< "$download_link" + count=${#pathElems[@]} + specific_version="${pathElems[count-2]}" + unset IFS; + say_verbose "Version: '$specific_version'." + + #Retrieve effective version + effective_version="$(get_specific_product_version "$azure_feed" "$specific_version" "$download_link")" + + # Add link info to arrays + download_links+=($download_link) + specific_versions+=($specific_version) + effective_versions+=($effective_version) + link_types+=("aka.ms") + + # Check if the SDK version is already installed. + if [[ "$dry_run" != true ]] && is_dotnet_package_installed "$install_root" "$asset_relative_path" "$effective_version"; then + say "$asset_name with version '$effective_version' is already installed." + exit 0 + fi + + return 0 + fi + + # if quality is specified - exit with error - there is no fallback approach + if [ ! -z "$normalized_quality" ]; then + say_err "Failed to locate the latest version in the channel '$normalized_channel' with '$normalized_quality' quality for '$normalized_product', os: '$normalized_os', architecture: '$normalized_architecture'." + say_err "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support." + return 1 + fi + say_verbose "Falling back to latest.version file approach." +} + +# THIS FUNCTION MAY EXIT (if the determined version is already installed) +# args: +# feed - $1 +generate_regular_links() { + local feed="$1" + local valid_legacy_download_link=true + + specific_version=$(get_specific_version_from_version "$feed" "$channel" "$normalized_architecture" "$version" "$json_file") || specific_version='0' + + if [[ "$specific_version" == '0' ]]; then + say_verbose "Failed to resolve the specific version number using feed '$feed'" + return + fi + + effective_version="$(get_specific_product_version "$feed" "$specific_version")" + say_verbose "specific_version=$specific_version" + + download_link="$(construct_download_link "$feed" "$channel" "$normalized_architecture" "$specific_version" "$normalized_os")" + say_verbose "Constructed primary named payload URL: $download_link" + + # Add link info to arrays + download_links+=($download_link) + specific_versions+=($specific_version) + effective_versions+=($effective_version) + link_types+=("primary") + + legacy_download_link="$(construct_legacy_download_link "$feed" "$channel" "$normalized_architecture" "$specific_version")" || valid_legacy_download_link=false + + if [ "$valid_legacy_download_link" = true ]; then + say_verbose "Constructed legacy named payload URL: $legacy_download_link" + + download_links+=($legacy_download_link) + specific_versions+=($specific_version) + effective_versions+=($effective_version) + link_types+=("legacy") + else + legacy_download_link="" + say_verbose "Cound not construct a legacy_download_link; omitting..." + fi + + # Check if the SDK version is already installed. + if [[ "$dry_run" != true ]] && is_dotnet_package_installed "$install_root" "$asset_relative_path" "$effective_version"; then + say "$asset_name with version '$effective_version' is already installed." + exit 0 + fi +} + +print_dry_run() { + + say "Payload URLs:" + + for link_index in "${!download_links[@]}" + do + say "URL #$link_index - ${link_types[$link_index]}: ${download_links[$link_index]}" + done + + resolved_version=${specific_versions[0]} + repeatable_command="./$script_name --version "\""$resolved_version"\"" --install-dir "\""$install_root"\"" --architecture "\""$normalized_architecture"\"" --os "\""$normalized_os"\""" + + if [ ! -z "$normalized_quality" ]; then + repeatable_command+=" --quality "\""$normalized_quality"\""" + fi + + if [[ "$runtime" == "dotnet" ]]; then + repeatable_command+=" --runtime "\""dotnet"\""" + elif [[ "$runtime" == "aspnetcore" ]]; then + repeatable_command+=" --runtime "\""aspnetcore"\""" + fi + + repeatable_command+="$non_dynamic_parameters" + + if [ -n "$feed_credential" ]; then + repeatable_command+=" --feed-credential "\"""\""" + fi + + say "Repeatable invocation: $repeatable_command" +} + +calculate_vars() { + eval $invocation + + script_name=$(basename "$0") + normalized_architecture="$(get_normalized_architecture_from_architecture "$architecture")" + say_verbose "Normalized architecture: '$normalized_architecture'." + normalized_os="$(get_normalized_os "$user_defined_os")" + say_verbose "Normalized OS: '$normalized_os'." + normalized_quality="$(get_normalized_quality "$quality")" + say_verbose "Normalized quality: '$normalized_quality'." + normalized_channel="$(get_normalized_channel "$channel")" + say_verbose "Normalized channel: '$normalized_channel'." + normalized_product="$(get_normalized_product "$runtime")" + say_verbose "Normalized product: '$normalized_product'." + install_root="$(resolve_installation_path "$install_dir")" + say_verbose "InstallRoot: '$install_root'." + + normalized_architecture="$(get_normalized_architecture_for_specific_sdk_version "$version" "$normalized_channel" "$normalized_architecture")" + + if [[ "$runtime" == "dotnet" ]]; then + asset_relative_path="shared/Microsoft.NETCore.App" + asset_name=".NET Core Runtime" + elif [[ "$runtime" == "aspnetcore" ]]; then + asset_relative_path="shared/Microsoft.AspNetCore.App" + asset_name="ASP.NET Core Runtime" + elif [ -z "$runtime" ]; then + asset_relative_path="sdk" + asset_name=".NET Core SDK" + fi + + get_feeds_to_use +} + +install_dotnet() { + eval $invocation + local download_failed=false + local download_completed=false + local remote_file_size=0 + + mkdir -p "$install_root" + zip_path="${zip_path:-$(mktemp "$temporary_file_template")}" + say_verbose "Zip path: $zip_path" + + for link_index in "${!download_links[@]}" + do + download_link="${download_links[$link_index]}" + specific_version="${specific_versions[$link_index]}" + effective_version="${effective_versions[$link_index]}" + link_type="${link_types[$link_index]}" + + say "Attempting to download using $link_type link $download_link" + + # The download function will set variables $http_code and $download_error_msg in case of failure. + download_failed=false + download "$download_link" "$zip_path" 2>&1 || download_failed=true + + if [ "$download_failed" = true ]; then + case $http_code in + 404) + say "The resource at $link_type link '$download_link' is not available." + ;; + *) + say "Failed to download $link_type link '$download_link': $download_error_msg" + ;; + esac + rm -f "$zip_path" 2>&1 && say_verbose "Temporary zip file $zip_path was removed" + else + download_completed=true + break + fi + done + + if [[ "$download_completed" == false ]]; then + say_err "Could not find \`$asset_name\` with version = $specific_version" + say_err "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" + return 1 + fi + + remote_file_size="$(get_remote_file_size "$download_link")" + + say "Extracting zip from $download_link" + extract_dotnet_package "$zip_path" "$install_root" "$remote_file_size" || return 1 + + # Check if the SDK version is installed; if not, fail the installation. + # if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed. + if [[ $specific_version == *"rtm"* || $specific_version == *"servicing"* ]]; then + IFS='-' + read -ra verArr <<< "$specific_version" + release_version="${verArr[0]}" + unset IFS; + say_verbose "Checking installation: version = $release_version" + if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$release_version"; then + say "Installed version is $effective_version" + return 0 + fi + fi + + # Check if the standard SDK version is installed. + say_verbose "Checking installation: version = $effective_version" + if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$effective_version"; then + say "Installed version is $effective_version" + return 0 + fi + + # Version verification failed. More likely something is wrong either with the downloaded content or with the verification algorithm. + say_err "Failed to verify the version of installed \`$asset_name\`.\nInstallation source: $download_link.\nInstallation location: $install_root.\nReport the bug at https://github.com/dotnet/install-scripts/issues." + say_err "\`$asset_name\` with version = $effective_version failed to install with an error." + return 1 +} + +args=("$@") + +local_version_file_relative_path="/.version" +bin_folder_relative_path="" +temporary_file_template="${TMPDIR:-/tmp}/dotnet.XXXXXXXXX" + +channel="LTS" +version="Latest" +json_file="" +install_dir="" +architecture="" +dry_run=false +no_path=false +no_cdn=false +azure_feed="" +uncached_feed="" +feed_credential="" +verbose=false +runtime="" +runtime_id="" +quality="" +internal=false +override_non_versioned_files=true +non_dynamic_parameters="" +user_defined_os="" + +while [ $# -ne 0 ] +do + name="$1" + case "$name" in + -c|--channel|-[Cc]hannel) + shift + channel="$1" + ;; + -v|--version|-[Vv]ersion) + shift + version="$1" + ;; + -q|--quality|-[Qq]uality) + shift + quality="$1" + ;; + --internal|-[Ii]nternal) + internal=true + non_dynamic_parameters+=" $name" + ;; + -i|--install-dir|-[Ii]nstall[Dd]ir) + shift + install_dir="$1" + ;; + --arch|--architecture|-[Aa]rch|-[Aa]rchitecture) + shift + architecture="$1" + ;; + --os|-[Oo][SS]) + shift + user_defined_os="$1" + ;; + --shared-runtime|-[Ss]hared[Rr]untime) + say_warning "The --shared-runtime flag is obsolete and may be removed in a future version of this script. The recommended usage is to specify '--runtime dotnet'." + if [ -z "$runtime" ]; then + runtime="dotnet" + fi + ;; + --runtime|-[Rr]untime) + shift + runtime="$1" + if [[ "$runtime" != "dotnet" ]] && [[ "$runtime" != "aspnetcore" ]]; then + say_err "Unsupported value for --runtime: '$1'. Valid values are 'dotnet' and 'aspnetcore'." + if [[ "$runtime" == "windowsdesktop" ]]; then + say_err "WindowsDesktop archives are manufactured for Windows platforms only." + fi + exit 1 + fi + ;; + --dry-run|-[Dd]ry[Rr]un) + dry_run=true + ;; + --no-path|-[Nn]o[Pp]ath) + no_path=true + non_dynamic_parameters+=" $name" + ;; + --verbose|-[Vv]erbose) + verbose=true + non_dynamic_parameters+=" $name" + ;; + --no-cdn|-[Nn]o[Cc]dn) + no_cdn=true + non_dynamic_parameters+=" $name" + ;; + --azure-feed|-[Aa]zure[Ff]eed) + shift + azure_feed="$1" + non_dynamic_parameters+=" $name "\""$1"\""" + ;; + --uncached-feed|-[Uu]ncached[Ff]eed) + shift + uncached_feed="$1" + non_dynamic_parameters+=" $name "\""$1"\""" + ;; + --feed-credential|-[Ff]eed[Cc]redential) + shift + feed_credential="$1" + #feed_credential should start with "?", for it to be added to the end of the link. + #adding "?" at the beginning of the feed_credential if needed. + [[ -z "$(echo $feed_credential)" ]] || [[ $feed_credential == \?* ]] || feed_credential="?$feed_credential" + ;; + --runtime-id|-[Rr]untime[Ii]d) + shift + runtime_id="$1" + non_dynamic_parameters+=" $name "\""$1"\""" + say_warning "Use of --runtime-id is obsolete and should be limited to the versions below 2.1. To override architecture, use --architecture option instead. To override OS, use --os option instead." + ;; + --jsonfile|-[Jj][Ss]on[Ff]ile) + shift + json_file="$1" + ;; + --skip-non-versioned-files|-[Ss]kip[Nn]on[Vv]ersioned[Ff]iles) + override_non_versioned_files=false + non_dynamic_parameters+=" $name" + ;; + --keep-zip|-[Kk]eep[Zz]ip) + keep_zip=true + non_dynamic_parameters+=" $name" + ;; + --zip-path|-[Zz]ip[Pp]ath) + shift + zip_path="$1" + ;; + -?|--?|-h|--help|-[Hh]elp) + script_name="$(basename "$0")" + echo ".NET Tools Installer" + echo "Usage:" + echo " # Install a .NET SDK of a given Quality from a given Channel" + echo " $script_name [-c|--channel ] [-q|--quality ]" + echo " # Install a .NET SDK of a specific public version" + echo " $script_name [-v|--version ]" + echo " $script_name -h|-?|--help" + echo "" + echo "$script_name is a simple command line interface for obtaining dotnet cli." + echo " Note that the intended use of this script is for Continuous Integration (CI) scenarios, where:" + echo " - The SDK needs to be installed without user interaction and without admin rights." + echo " - The SDK installation doesn't need to persist across multiple CI runs." + echo " To set up a development environment or to run apps, use installers rather than this script. Visit https://dotnet.microsoft.com/download to get the installer." + echo "" + echo "Options:" + echo " -c,--channel Download from the channel specified, Defaults to \`$channel\`." + echo " -Channel" + echo " Possible values:" + echo " - STS - the most recent Standard Term Support release" + echo " - LTS - the most recent Long Term Support release" + echo " - 2-part version in a format A.B - represents a specific release" + echo " examples: 2.0; 1.0" + echo " - 3-part version in a format A.B.Cxx - represents a specific SDK release" + echo " examples: 5.0.1xx, 5.0.2xx." + echo " Supported since 5.0 release" + echo " Warning: Value 'Current' is deprecated for the Channel parameter. Use 'STS' instead." + echo " Note: The version parameter overrides the channel parameter when any version other than 'latest' is used." + echo " -v,--version Use specific VERSION, Defaults to \`$version\`." + echo " -Version" + echo " Possible values:" + echo " - latest - the latest build on specific channel" + echo " - 3-part version in a format A.B.C - represents specific version of build" + echo " examples: 2.0.0-preview2-006120; 1.1.0" + echo " -q,--quality Download the latest build of specified quality in the channel." + echo " -Quality" + echo " The possible values are: daily, signed, validated, preview, GA." + echo " Works only in combination with channel. Not applicable for STS and LTS channels and will be ignored if those channels are used." + echo " For SDK use channel in A.B.Cxx format. Using quality for SDK together with channel in A.B format is not supported." + echo " Supported since 5.0 release." + echo " Note: The version parameter overrides the channel parameter when any version other than 'latest' is used, and therefore overrides the quality." + echo " --internal,-Internal Download internal builds. Requires providing credentials via --feed-credential parameter." + echo " --feed-credential Token to access Azure feed. Used as a query string to append to the Azure feed." + echo " -FeedCredential This parameter typically is not specified." + echo " -i,--install-dir Install under specified location (see Install Location below)" + echo " -InstallDir" + echo " --architecture Architecture of dotnet binaries to be installed, Defaults to \`$architecture\`." + echo " --arch,-Architecture,-Arch" + echo " Possible values: x64, arm, arm64, s390x, ppc64le and loongarch64" + echo " --os Specifies operating system to be used when selecting the installer." + echo " Overrides the OS determination approach used by the script. Supported values: osx, linux, linux-musl, freebsd, rhel.6." + echo " In case any other value is provided, the platform will be determined by the script based on machine configuration." + echo " Not supported for legacy links. Use --runtime-id to specify platform for legacy links." + echo " Refer to: https://aka.ms/dotnet-os-lifecycle for more information." + echo " --runtime Installs a shared runtime only, without the SDK." + echo " -Runtime" + echo " Possible values:" + echo " - dotnet - the Microsoft.NETCore.App shared runtime" + echo " - aspnetcore - the Microsoft.AspNetCore.App shared runtime" + echo " --dry-run,-DryRun Do not perform installation. Display download link." + echo " --no-path, -NoPath Do not set PATH for the current process." + echo " --verbose,-Verbose Display diagnostics information." + echo " --azure-feed,-AzureFeed For internal use only." + echo " Allows using a different storage to download SDK archives from." + echo " This parameter is only used if --no-cdn is false." + echo " --uncached-feed,-UncachedFeed For internal use only." + echo " Allows using a different storage to download SDK archives from." + echo " This parameter is only used if --no-cdn is true." + echo " --skip-non-versioned-files Skips non-versioned files if they already exist, such as the dotnet executable." + echo " -SkipNonVersionedFiles" + echo " --no-cdn,-NoCdn Disable downloading from the Azure CDN, and use the uncached feed directly." + echo " --jsonfile Determines the SDK version from a user specified global.json file." + echo " Note: global.json must have a value for 'SDK:Version'" + echo " --keep-zip,-KeepZip If set, downloaded file is kept." + echo " --zip-path, -ZipPath If set, downloaded file is stored at the specified path." + echo " -?,--?,-h,--help,-Help Shows this help message" + echo "" + echo "Install Location:" + echo " Location is chosen in following order:" + echo " - --install-dir option" + echo " - Environmental variable DOTNET_INSTALL_DIR" + echo " - $HOME/.dotnet" + exit 0 + ;; + *) + say_err "Unknown argument \`$name\`" + exit 1 + ;; + esac + + shift +done + +say_verbose "Note that the intended use of this script is for Continuous Integration (CI) scenarios, where:" +say_verbose "- The SDK needs to be installed without user interaction and without admin rights." +say_verbose "- The SDK installation doesn't need to persist across multiple CI runs." +say_verbose "To set up a development environment or to run apps, use installers rather than this script. Visit https://dotnet.microsoft.com/download to get the installer.\n" + +if [ "$internal" = true ] && [ -z "$(echo $feed_credential)" ]; then + message="Provide credentials via --feed-credential parameter." + if [ "$dry_run" = true ]; then + say_warning "$message" + else + say_err "$message" + exit 1 + fi +fi + +check_min_reqs +calculate_vars +# generate_regular_links call below will 'exit' if the determined version is already installed. +generate_download_links + +if [[ "$dry_run" = true ]]; then + print_dry_run + exit 0 +fi + +install_dotnet + +bin_path="$(get_absolute_path "$(combine_paths "$install_root" "$bin_folder_relative_path")")" +if [ "$no_path" = false ]; then + say "Adding to current process PATH: \`$bin_path\`. Note: This change will be visible only when sourcing script." + export PATH="$bin_path":"$PATH" +else + say "Binaries of dotnet can be found in $bin_path" +fi + +say "Note that the script does not resolve dependencies during installation." +say "To check the list of dependencies, go to https://learn.microsoft.com/dotnet/core/install, select your operating system and check the \"Dependencies\" section." +say "Installation finished successfully." From 4db22d38c6f04996de99ff7062555e0eb2f66b9c Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 27 Feb 2024 12:49:38 -0800 Subject: [PATCH 008/247] Docker-in-docker: Updates default value for "dockerDashComposeVersion" to "latest (#873) * Set v2 by default ; draft * Fix edge case: compose switch * Update description * validate "compose-switch" installation * fix test --- .../devcontainer-feature.json | 13 +- src/docker-in-docker/install.sh | 113 +++++++++--------- test/docker-in-docker/docker_build.sh | 4 +- test/docker-in-docker/docker_compose_v1.sh | 13 ++ test/docker-in-docker/docker_compose_v2.sh | 16 +++ test/docker-in-docker/scenarios.json | 20 ++++ 6 files changed, 119 insertions(+), 60 deletions(-) create mode 100755 test/docker-in-docker/docker_compose_v1.sh create mode 100755 test/docker-in-docker/docker_compose_v2.sh diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index d74f205a1..ab26a5e11 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.9.2", + "version": "2.10.0", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", @@ -29,11 +29,11 @@ "type": "string", "enum": [ "none", - "v1", + "latest", "v2" ], - "default": "v1", - "description": "Default version of Docker Compose (v1 or v2 or none)" + "default": "latest", + "description": "Default version of Docker Compose (latest, v2 or none)" }, "azureDnsAutoDetection": { "type": "boolean", @@ -50,6 +50,11 @@ "type": "boolean", "default": true, "description": "Install Docker Buildx" + }, + "installDockerComposeSwitch": { + "type": "boolean", + "default": true, + "description": "Install Compose Switch (provided docker compose is available) which is a replacement to the Compose V1 docker-compose (python) executable. It translates the command line into Compose V2 docker compose then runs the latter." } }, "entrypoint": "/usr/local/share/docker-init.sh", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index 9f0e18cee..2a27f5c9c 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -11,11 +11,12 @@ DOCKER_VERSION="${VERSION:-"latest"}" # The Docker/Moby Engine + CLI should match in version USE_MOBY="${MOBY:-"true"}" MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"latest"}" -DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v1"}" # v1 or v2 or none +DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"latest"}" #latest, v2 or none AZURE_DNS_AUTO_DETECTION="${AZUREDNSAUTODETECTION:-"true"}" DOCKER_DEFAULT_ADDRESS_POOL="${DOCKERDEFAULTADDRESSPOOL:-""}" USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" INSTALL_DOCKER_BUILDX="${INSTALLDOCKERBUILDX:-"true"}" +INSTALL_DOCKER_COMPOSE_SWITCH="${INSTALLDOCKERCOMPOSESWITCH:-"true"}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal jammy" DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal hirsute impish jammy" @@ -247,75 +248,77 @@ fi echo "Finished installing docker / moby!" +docker_home="/usr/libexec/docker" +cli_plugins_dir="${docker_home}/cli-plugins" + # If 'docker-compose' command is to be included if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then - # Install Docker Compose if not already installed and is on a supported architecture - if type docker-compose > /dev/null 2>&1; then - echo "Docker Compose v1 already installed." - else - target_compose_arch="${architecture}" - if [ "${target_compose_arch}" = "amd64" ]; then - target_compose_arch="x86_64" - fi - # https://github.com/devcontainers/features/issues/832 - if [ "${target_compose_arch}" != "x86_64" ] && [ "${VERSION_CODENAME}" != "bookworm" ]; then - # Use pip to get a version that runs on this architecture - check_packages python3-minimal python3-pip libffi-dev python3-venv - export PIPX_HOME=/usr/local/pipx - mkdir -p ${PIPX_HOME} - export PIPX_BIN_DIR=/usr/local/bin - export PYTHONUSERBASE=/tmp/pip-tmp - export PIP_CACHE_DIR=/tmp/pip-tmp/cache - pipx_bin=pipx - if ! type pipx > /dev/null 2>&1; then - pip3 install --disable-pip-version-check --no-cache-dir --user pipx - pipx_bin=/tmp/pip-tmp/bin/pipx - fi - - set +e - ${pipx_bin} install --pip-args '--no-cache-dir --force-reinstall' docker-compose - exit_code=$? - set -e - - if [ ${exit_code} -ne 0 ]; then - # Temporary: https://github.com/devcontainers/features/issues/616 - # See https://github.com/yaml/pyyaml/issues/601 - echo "(*) Failed to install docker-compose via pipx. Trying via pip3..." - - export PYTHONUSERBASE=/usr/local - pip3 install --disable-pip-version-check --no-cache-dir --user "Cython<3.0" pyyaml wheel docker-compose --no-build-isolation - fi + case "${architecture}" in + amd64) target_compose_arch=x86_64 ;; + arm64) target_compose_arch=aarch64 ;; + *) + echo "(!) Docker in docker does not support machine architecture '$architecture'. Please use an x86-64 or ARM64 machine." + exit 1 + esac - rm -rf /tmp/pip-tmp + docker_compose_path="/usr/local/bin/docker-compose" + if [ "${DOCKER_DASH_COMPOSE_VERSION}" = "v1" ]; then + err "The final Compose V1 release, version 1.29.2, was May 10, 2021. These packages haven't received any security updates since then. Use at your own risk." + INSTALL_DOCKER_COMPOSE_SWITCH="false" + + if [ "${target_compose_arch}" = "x86_64" ]; then + echo "(*) Installing docker compose v1..." + curl -fsSL "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-Linux-x86_64" -o ${docker_compose_path} + chmod +x ${docker_compose_path} + + # Download the SHA256 checksum + DOCKER_COMPOSE_SHA256="$(curl -sSL "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-Linux-x86_64.sha256" | awk '{print $1}')" + echo "${DOCKER_COMPOSE_SHA256} ${docker_compose_path}" > docker-compose.sha256sum + sha256sum -c docker-compose.sha256sum --ignore-missing + elif [ "${VERSION_CODENAME}" = "bookworm" ]; then + err "Docker compose v1 is unavailable for 'bookworm' on Arm64. Kindly switch to use v2" + exit 1 else - compose_v1_version="1" - find_version_from_git_tags compose_v1_version "https://github.com/docker/compose" "tags/" - echo "(*) Installing docker-compose ${compose_v1_version}..." - curl -fsSL "https://github.com/docker/compose/releases/download/${compose_v1_version}/docker-compose-Linux-x86_64" -o /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose + # Use pip to get a version that runs on this architecture + check_packages python3-minimal python3-pip libffi-dev python3-venv + echo "(*) Installing docker compose v1 via pip..." + export PYTHONUSERBASE=/usr/local + pip3 install --disable-pip-version-check --no-cache-dir --user "Cython<3.0" pyyaml wheel docker-compose --no-build-isolation fi + else + compose_version=${DOCKER_DASH_COMPOSE_VERSION#v} + find_version_from_git_tags compose_version "https://github.com/docker/compose" "tags/v" + echo "(*) Installing docker-compose ${compose_version}..." + curl -L "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} + chmod +x ${docker_compose_path} + + # Download the SHA256 checksum + DOCKER_COMPOSE_SHA256="$(curl -sSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}.sha256" | awk '{print $1}')" + echo "${DOCKER_COMPOSE_SHA256} ${docker_compose_path}" > docker-compose.sha256sum + sha256sum -c docker-compose.sha256sum --ignore-missing + + mkdir -p ${cli_plugins_dir} + cp ${docker_compose_path} ${cli_plugins_dir} fi +fi - # Install docker-compose switch if not already installed - https://github.com/docker/compose-switch#manual-installation - current_v1_compose_path="$(which docker-compose)" - target_v1_compose_path="$(dirname "${current_v1_compose_path}")/docker-compose-v1" - if ! type compose-switch > /dev/null 2>&1; then +# Install docker-compose switch if not already installed - https://github.com/docker/compose-switch#manual-installation +if [ "${INSTALL_DOCKER_COMPOSE_SWITCH}" = "true" ] && ! type compose-switch > /dev/null 2>&1; then + if type docker-compose > /dev/null 2>&1; then echo "(*) Installing compose-switch..." + current_compose_path="$(which docker-compose)" + target_compose_path="$(dirname "${current_compose_path}")/docker-compose-v1" compose_switch_version="latest" find_version_from_git_tags compose_switch_version "https://github.com/docker/compose-switch" curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch chmod +x /usr/local/bin/compose-switch # TODO: Verify checksum once available: https://github.com/docker/compose-switch/issues/11 - # Setup v1 CLI as alternative in addition to compose-switch (which maps to v2) - mv "${current_v1_compose_path}" "${target_v1_compose_path}" - update-alternatives --install /usr/local/bin/docker-compose docker-compose /usr/local/bin/compose-switch 99 - update-alternatives --install /usr/local/bin/docker-compose docker-compose "${target_v1_compose_path}" 1 - fi - if [ "${DOCKER_DASH_COMPOSE_VERSION}" = "v1" ]; then - update-alternatives --set docker-compose "${target_v1_compose_path}" + mv "${current_compose_path}" "${target_compose_path}" + update-alternatives --install ${docker_compose_path} docker-compose /usr/local/bin/compose-switch 99 + update-alternatives --install ${docker_compose_path} docker-compose "${target_compose_path}" 1 else - update-alternatives --set docker-compose /usr/local/bin/compose-switch + err "Skipping installation of compose-switch as docker compose is unavailable..." fi fi diff --git a/test/docker-in-docker/docker_build.sh b/test/docker-in-docker/docker_build.sh index 7e71be8a1..322a819fc 100755 --- a/test/docker-in-docker/docker_build.sh +++ b/test/docker-in-docker/docker_build.sh @@ -9,8 +9,10 @@ source dev-container-features-test-lib check "docker-buildx" docker buildx version check "docker-build" docker build ./ -check "installs docker-compose v1 install" bash -c "type docker-compose" check "installs compose-switch" bash -c "[[ -f /usr/local/bin/compose-switch ]]" +check "docker compose" bash -c "docker compose version | grep -E '2.[0-9]+.[0-9]+'" +check "docker-compose" bash -c "docker-compose --version | grep -E '2.[0-9]+.[0-9]+'" + check "docker-buildx" bash -c "docker buildx version" check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" diff --git a/test/docker-in-docker/docker_compose_v1.sh b/test/docker-in-docker/docker_compose_v1.sh new file mode 100755 index 000000000..3f7453c83 --- /dev/null +++ b/test/docker-in-docker/docker_compose_v1.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests + +check "docker-compose" bash -c "docker-compose version | grep -E '1.[0-9]+.[0-9]+'" + +# Report result +reportResults diff --git a/test/docker-in-docker/docker_compose_v2.sh b/test/docker-in-docker/docker_compose_v2.sh new file mode 100755 index 000000000..5a512d2c5 --- /dev/null +++ b/test/docker-in-docker/docker_compose_v2.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Definition specific tests + +check "docker compose" bash -c "docker compose version | grep -E '2.[0-9]+.[0-9]+'" +check "docker-compose" bash -c "docker-compose --version | grep -E '2.[0-9]+.[0-9]+'" +check "installs compose-switch as docker-compose" bash -c "[[ -f /usr/local/bin/docker-compose ]]" +check "installs compose-switch" bash -c "[[ -f /usr/local/bin/compose-switch ]]" + +# Report result +reportResults diff --git a/test/docker-in-docker/scenarios.json b/test/docker-in-docker/scenarios.json index ccf57b188..e209f3453 100644 --- a/test/docker-in-docker/scenarios.json +++ b/test/docker-in-docker/scenarios.json @@ -87,6 +87,26 @@ } } }, + "docker_compose_v1": { + "image": "mcr.microsoft.com/devcontainers/base:focal", + "features": { + "docker-in-docker": { + "moby": true, + "installDockerBuildx": true, + "dockerDashComposeVersion": "v1" + } + } + }, + "docker_compose_v2": { + "image": "mcr.microsoft.com/devcontainers/base:focal", + "features": { + "docker-in-docker": { + "moby": true, + "installDockerBuildx": true, + "dockerDashComposeVersion": "v2" + } + } + }, "docker_build_fallback_buildx": { "image": "ubuntu:focal", "features": { From 03806caa7b6747489311e1f4d8f912ee456039e6 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Tue, 27 Feb 2024 12:50:57 -0800 Subject: [PATCH 009/247] Automated documentation update (#886) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/docker-in-docker/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/docker-in-docker/README.md b/src/docker-in-docker/README.md index 2ab700118..b913fa45f 100644 --- a/src/docker-in-docker/README.md +++ b/src/docker-in-docker/README.md @@ -18,10 +18,11 @@ Create child containers *inside* a container, independent from the host's docker | version | Select or enter a Docker/Moby Engine version. (Availability can vary by OS version.) | string | latest | | moby | Install OSS Moby build instead of Docker CE | boolean | true | | mobyBuildxVersion | Install a specific version of moby-buildx when using Moby | string | latest | -| dockerDashComposeVersion | Default version of Docker Compose (v1 or v2 or none) | string | v1 | +| dockerDashComposeVersion | Default version of Docker Compose (latest, v2 or none) | string | latest | | azureDnsAutoDetection | Allow automatically setting the dockerd DNS server when the installation script detects it is running in Azure | boolean | true | | dockerDefaultAddressPool | Define default address pools for Docker networks. e.g. base=192.168.0.0/16,size=24 | string | - | | installDockerBuildx | Install Docker Buildx | boolean | true | +| installDockerComposeSwitch | Install Compose Switch (provided docker compose is available) which is a replacement to the Compose V1 docker-compose (python) executable. It translates the command line into Compose V2 docker compose then runs the latter. | boolean | true | ## Customizations From 5ca92d26ce59ba17e9c5c67fb265b3694c4f18c3 Mon Sep 17 00:00:00 2001 From: gauravsaini04 <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 29 Feb 2024 00:25:54 +0530 Subject: [PATCH 010/247] [kubectl-helm-minikube]: Handle failing build with fallback to prev. version for helm (#875) * [kubectl-helm-minikube]: Handle failing build with fallback to prev. version * changes acc. to comments and added tests * only to add fallback mechanism for helm and removed for kubectl & minikube * bump the patch version * changes for last comments by @samruddhikhandale * changes for suggestions --- .../devcontainer-feature.json | 2 +- src/kubectl-helm-minikube/install.sh | 28 +++- .../install_only_helm_fallback.sh | 124 ++++++++++++++++++ test/kubectl-helm-minikube/scenarios.json | 10 ++ 4 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 test/kubectl-helm-minikube/install_only_helm_fallback.sh diff --git a/src/kubectl-helm-minikube/devcontainer-feature.json b/src/kubectl-helm-minikube/devcontainer-feature.json index fa8ea6138..16472205f 100644 --- a/src/kubectl-helm-minikube/devcontainer-feature.json +++ b/src/kubectl-helm-minikube/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "kubectl-helm-minikube", - "version": "1.1.6", + "version": "1.1.7", "name": "Kubectl, Helm, and Minikube", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/kubectl-helm-minikube", "description": "Installs latest version of kubectl, Helm, and optionally minikube. Auto-detects latest versions and installs needed dependencies.", diff --git a/src/kubectl-helm-minikube/install.sh b/src/kubectl-helm-minikube/install.sh index 5deb377c3..69d4e0237 100755 --- a/src/kubectl-helm-minikube/install.sh +++ b/src/kubectl-helm-minikube/install.sh @@ -155,6 +155,21 @@ if [ ${KUBECTL_VERSION} != "none" ]; then fi fi +# Function to fetch the version released prior to the latest version +get_previous_version() { + repo_url=$1 + # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' +} + +get_helm() { + HELM_VERSION=$1 + helm_filename="helm-${HELM_VERSION}-linux-${architecture}.tar.gz" + tmp_helm_filename="/tmp/helm/${helm_filename}" + curl -sSL "https://get.helm.sh/${helm_filename}" -o "${tmp_helm_filename}" + curl -sSL "https://github.com/helm/helm/releases/download/${HELM_VERSION}/${helm_filename}.asc" -o "${tmp_helm_filename}.asc" +} + if [ ${HELM_VERSION} != "none" ]; then # Install Helm, verify signature and checksum echo "Downloading Helm..." @@ -163,10 +178,15 @@ if [ ${HELM_VERSION} != "none" ]; then HELM_VERSION="v${HELM_VERSION}" fi mkdir -p /tmp/helm - helm_filename="helm-${HELM_VERSION}-linux-${architecture}.tar.gz" - tmp_helm_filename="/tmp/helm/${helm_filename}" - curl -sSL "https://get.helm.sh/${helm_filename}" -o "${tmp_helm_filename}" - curl -sSL "https://github.com/helm/helm/releases/download/${HELM_VERSION}/${helm_filename}.asc" -o "${tmp_helm_filename}.asc" + get_helm "${HELM_VERSION}" + if grep -q "BlobNotFound" "${tmp_helm_filename}"; then + echo -e "\n(!) Failed to fetch the latest artifacts for helm ${HELM_VERSION}..." + repo_url=https://api.github.com/repos/helm/helm/releases + requested_version=$(get_previous_version "${repo_url}") + echo -e "\nAttempting to install ${requested_version}" + HELM_VERSION=${requested_version} + get_helm "${HELM_VERSION}" + fi export GNUPGHOME="/tmp/helm/gnupg" mkdir -p "${GNUPGHOME}" chmod 700 ${GNUPGHOME} diff --git a/test/kubectl-helm-minikube/install_only_helm_fallback.sh b/test/kubectl-helm-minikube/install_only_helm_fallback.sh new file mode 100644 index 000000000..58ed137de --- /dev/null +++ b/test/kubectl-helm-minikube/install_only_helm_fallback.sh @@ -0,0 +1,124 @@ +#!/bin/bash +set -e + +# Optional: Import test library +source dev-container-features-test-lib +HL="\033[1;33m" +N="\033[0;37m" +echo -e "\n๐Ÿ‘‰${HL} helm version as installed by kubectl-helm-minikube feature${N}:" + +set +e + check "helm version" helm version +set -e + +# Function to handle errors +handle_error() { + local exit_code=$? + local line_number=$1 + local command=$2 + echo "Error occurred at line $line_number with exit code $exit_code in command $command" + exit $exit_code +} +trap 'handle_error $LINENO ${BASH_COMMAND%% *}' ERR +echo "This is line $LINENO" + +## Check for fallback version installation instead of latest ( when artifact not found ) +architecture="$(uname -m)" +case $architecture in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture $architecture unsupported"; exit 1 ;; +esac +HELM_SHA256="${HELM_SHA256:-"automatic"}" +HELM_GPG_KEYS_URI="https://raw.githubusercontent.com/helm/helm/main/KEYS" + +repo_url=https://api.github.com/repos/helm/helm/releases + +# Function to fetch the latest version of the plugin +get_latest_version() { + curl -s "$repo_url/latest" | jq -r '.tag_name' +} + +# Function to change the patch number in a semver version +change_patch_number() { + local version="$1" # Input version + local new_patch="$2" # New patch number + # Extract major, minor, and current patch numbers + local major=$(echo "$version" | cut -d. -f1) + local minor=$(echo "$version" | cut -d. -f2) + local current_patch=$(echo "$version" | cut -d. -f3) + # Construct the new version with the updated patch number + local new_version="$major.$minor.$new_patch" + echo "$new_version" +} + +# Function to fetch the previous version of the plugin +get_previous_version() { + # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' +} + +get_helm() { + HELM_VERSION=$1 + helm_filename="helm-${HELM_VERSION}-linux-${architecture}.tar.gz" + tmp_helm_filename="/tmp/helm/${helm_filename}" + sudo curl -sSL "https://get.helm.sh/${helm_filename}" -o "${tmp_helm_filename}" + sudo curl -sSL "https://github.com/helm/helm/releases/download/${HELM_VERSION}/${helm_filename}.asc" -o "${tmp_helm_filename}.asc" +} + +latest_version=$(get_latest_version) +NON_EXISTING_PATCH_VERSION="xyz" +HELM_VERSION="$(change_patch_number ${latest_version} ${NON_EXISTING_PATCH_VERSION})" +echo -e "\n๐Ÿ‘‰${HL} Trying to install HELM_VERSION = ${HELM_VERSION}${N}"; +sudo mkdir -p /tmp/helm +get_helm "${HELM_VERSION}" +if grep -q "BlobNotFound" "/tmp/helm/${helm_filename}"; then + echo -e "\n(!) Failed to fetch the latest artifacts for helm ${HELM_VERSION}..." + requested_version=$(get_previous_version) + echo -e "\nAttempting to install ${requested_version}" + HELM_VERSION=${requested_version} + get_helm "${HELM_VERSION}" +fi +export GNUPGHOME="/tmp/helm/gnupg" +sudo mkdir -p "${GNUPGHOME}" +sudo chmod 700 ${GNUPGHOME} +sudo curl -sSL "${HELM_GPG_KEYS_URI}" -o /tmp/helm/KEYS +sudo echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" | sudo tee ${GNUPGHOME}/dirmngr.conf >/dev/null +sudo gpg -q --import "/tmp/helm/KEYS" +if ! sudo gpg --verify "${tmp_helm_filename}.asc" | sudo tee ${GNUPGHOME}/verify.log 2>&1; then + echo "Verification failed!" + sudo cat /tmp/helm/gnupg/verify.log + exit 1 +fi + +if [ "${HELM_SHA256}" = "automatic" ]; then + sudo curl -sSL "https://get.helm.sh/${helm_filename}.sha256" -o "${tmp_helm_filename}.sha256" + sudo curl -sSL "https://github.com/helm/helm/releases/download/${HELM_VERSION}/${helm_filename}.sha256.asc" -o "${tmp_helm_filename}.sha256.asc" + if ! sudo gpg --verify "${tmp_helm_filename}.sha256.asc" | sudo tee /tmp/helm/gnupg/verify.log 2>&1; then + echo "Verification failed!" + sudo cat /tmp/helm/gnupg/verify.log + exit 1 + fi + HELM_SHA256="$(sudo cat "${tmp_helm_filename}.sha256")" +fi + +([ "${HELM_SHA256}" = "dev-mode" ] || (sudo echo "${HELM_SHA256} *${tmp_helm_filename}" | sha256sum -c -)) +sudo tar xf "${tmp_helm_filename}" -C /tmp/helm +sudo mv -f "/tmp/helm/linux-${architecture}/helm" /usr/local/bin/ +sudo chmod 0755 /usr/local/bin/helm +sudo rm -rf /tmp/helm +if ! type helm > /dev/null 2>&1; then + echo '(!) Helm installation failed!' + exit 1 +fi + +echo -e "\n๐Ÿ‘‰${HL} helm version as installed by test for fallback${N}:" + +set +e + check "helm version" helm version +set -e + +# Report result +reportResults diff --git a/test/kubectl-helm-minikube/scenarios.json b/test/kubectl-helm-minikube/scenarios.json index 08ff8cc79..b82a9966e 100644 --- a/test/kubectl-helm-minikube/scenarios.json +++ b/test/kubectl-helm-minikube/scenarios.json @@ -8,5 +8,15 @@ "minikube": "none" } } + }, + "install_only_helm_fallback": { + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", + "features": { + "kubectl-helm-minikube": { + "version": "none", + "helm": "latest", + "minikube": "none" + } + } } } From 025cae18394696e5684251fbf72710e523c4bb65 Mon Sep 17 00:00:00 2001 From: Nebula <40148908+nebula-it@users.noreply.github.com> Date: Thu, 29 Feb 2024 17:13:15 -0800 Subject: [PATCH 011/247] Fix profile not downloading (#876) * Fix profile not downloading Since the powershell folder is owned by root, the profile download fails without becoming sudo first. ``` ls -l /opt/microsoft/powershell/ total 0 drwxr-xr-x. 1 root root 22 Feb 23 08:37 7 ``` * Update install.sh * Update install.sh * Fix: Add missing quote * Update install.sh * Update src/powershell/install.sh Co-authored-by: Samruddhi Khandale * Update install.sh * Update patch version in devcontainer-feature.json --------- Co-authored-by: Samruddhi Khandale --- src/powershell/devcontainer-feature.json | 2 +- src/powershell/install.sh | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index 5a831f7e3..289bd33f0 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.3.1", + "version": "1.3.2", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/powershell/install.sh b/src/powershell/install.sh index 2c052d7cd..27e5d8a27 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -12,6 +12,8 @@ set -e # Clean up rm -rf /var/lib/apt/lists/* +USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" + POWERSHELL_VERSION=${VERSION:-"latest"} POWERSHELL_MODULES="${MODULES:-""}" POWERSHELL_PROFILE_URL="${PROFILE_URL}" @@ -163,10 +165,26 @@ if [ ${#POWERSHELL_MODULES[@]} -gt 0 ]; then done fi + # Determine the appropriate non-root user + if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then + USERNAME="" + POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") + for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do + if id -u ${CURRENT_USER} > /dev/null 2>&1; then + USERNAME=${CURRENT_USER} + break + fi + done + if [ "${USERNAME}" = "" ]; then + USERNAME=root + fi + elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then + USERNAME=root + fi # If URL for powershell profile is provided, download it to '/opt/microsoft/powershell/7/profile.ps1' if [ -n "$POWERSHELL_PROFILE_URL" ]; then echo "Downloading PowerShell Profile from: $POWERSHELL_PROFILE_URL" - curl -sSL -o "/opt/microsoft/powershell/7/profile.ps1" "$POWERSHELL_PROFILE_URL" + su ${USERNAME} -c "curl -sSL -o '/opt/microsoft/powershell/7/profile.ps1' '$POWERSHELL_PROFILE_URL'" fi # Clean up From 084df810aed51ae4a3de9b352a57a9890d5e967d Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Mon, 4 Mar 2024 11:36:03 -0800 Subject: [PATCH 012/247] Oryx: Clean up additional installation of .NET (#892) --- src/oryx/devcontainer-feature.json | 2 +- src/oryx/install.sh | 14 +++++++++----- test/oryx/install_dotnet_and_oryx.sh | 3 +++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/oryx/devcontainer-feature.json b/src/oryx/devcontainer-feature.json index fe13e2fcb..a075096bf 100644 --- a/src/oryx/devcontainer-feature.json +++ b/src/oryx/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "oryx", - "version": "1.3.0", + "version": "1.3.1", "name": "Oryx", "description": "Installs the oryx CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/oryx", diff --git a/src/oryx/install.sh b/src/oryx/install.sh index f4c547fb5..407406141 100755 --- a/src/oryx/install.sh +++ b/src/oryx/install.sh @@ -86,7 +86,6 @@ install_dotnet_with_script() DOTNET_BINARY="dotnet" export PATH="${PATH}:/usr/share/dotnet" - DOTNET_BINARY_INSTALLATION="/usr/share/dotnet/sdk/${version}" } install_dotnet_using_apt() { @@ -146,7 +145,6 @@ usermod -a -G oryx "${USERNAME}" # Required to decide if we want to clean up dotnet later. DOTNET_INSTALLATION_PACKAGE="" -DOTNET_BINARY_INSTALLATION="" DOTNET_BINARY="" if dotnet --version > /dev/null ; then @@ -156,6 +154,7 @@ fi MAJOR_VERSION_ID=$(echo $(dotnet --version) | cut -d . -f 1) PATCH_VERSION_ID=$(echo $(dotnet --version) | cut -d . -f 3) +PINNED_SDK_VERSION="" # Oryx needs to be built with .NET 8 if [[ "${DOTNET_BINARY}" = "" ]] || [[ $MAJOR_VERSION_ID != "8" ]] || [[ $MAJOR_VERSION_ID = "8" && ${PATCH_VERSION_ID} -ge "101" ]] ; then echo "'dotnet 8' was not detected. Attempting to install .NET 8 to build oryx." @@ -182,7 +181,7 @@ mkdir -p ${ORYX} git clone --depth=1 https://github.com/microsoft/Oryx $GIT_ORYX -if [[ "${DOTNET_BINARY_INSTALLATION}" != "" ]]; then +if [[ "${PINNED_SDK_VERSION}" != "" ]]; then cd $GIT_ORYX dotnet new globaljson --sdk-version ${PINNED_SDK_VERSION} fi @@ -234,9 +233,14 @@ if [[ "${DOTNET_INSTALLATION_PACKAGE}" != "" ]]; then apt purge -yq $DOTNET_INSTALLATION_PACKAGE fi -if [[ "${DOTNET_BINARY_INSTALLATION}" != "" ]]; then +if [[ "${PINNED_SDK_VERSION}" != "" ]]; then rm -f ${GIT_ORYX}/global.json - rm -rf ${DOTNET_BINARY_INSTALLATION} + rm -rf /usr/share/dotnet/sdk/$PINNED_SDK_VERSION + + # Extract the major, minor version and the first digit of the patch version + MAJOR_MINOR_PATCH1_VERSION=${PINNED_SDK_VERSION%??} + rm -rf /usr/share/dotnet/shared/Microsoft.NETCore.App/$MAJOR_MINOR_PATCH1_VERSION + rm -rf /usr/share/dotnet/shared/Microsoft.AspNetCore.App/$MAJOR_MINOR_PATCH1_VERSION fi diff --git a/test/oryx/install_dotnet_and_oryx.sh b/test/oryx/install_dotnet_and_oryx.sh index da0a9162d..670d7565a 100644 --- a/test/oryx/install_dotnet_and_oryx.sh +++ b/test/oryx/install_dotnet_and_oryx.sh @@ -5,6 +5,9 @@ set -e # Optional: Import test library source dev-container-features-test-lib +# Runtimes are listed twice due to 'Microsoft.NETCore.App' and 'Microsoft.AspNetCore.App' +check "two versions of dotnet runtimes are present" bash -c "[ $(dotnet --list-runtimes | wc -l) -eq 4 ]" + check "Oryx version" oryx --version check "Dotnet is not removed if it is not installed by the Oryx Feature" dotnet --version From 4d2e62e4ed458469fd81b71ad3f4d8e1c5596b6f Mon Sep 17 00:00:00 2001 From: Nebula <40148908+nebula-it@users.noreply.github.com> Date: Mon, 4 Mar 2024 16:07:37 -0800 Subject: [PATCH 013/247] Fix PowerShell profile not loading (#889) * Update devcontainer-feature.json * Fix profile loading * Update devcontainer-feature.json * Update devcontainer-feature.json * Add -E to sudo so it inherits environment --- src/powershell/devcontainer-feature.json | 4 ++-- src/powershell/install.sh | 25 +++++------------------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index 289bd33f0..86314be64 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.3.2", + "version": "1.3.3", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", @@ -20,7 +20,7 @@ "default": "", "description": "Optional comma separated list of PowerShell modules to install." }, - "powershellProfileURL ": { + "powershellProfileURL": { "type": "string", "default": "", "description": "Optional (publicly accessible) URL to download PowerShell profile." diff --git a/src/powershell/install.sh b/src/powershell/install.sh index 27e5d8a27..0ce63c95d 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -12,11 +12,9 @@ set -e # Clean up rm -rf /var/lib/apt/lists/* -USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" - POWERSHELL_VERSION=${VERSION:-"latest"} POWERSHELL_MODULES="${MODULES:-""}" -POWERSHELL_PROFILE_URL="${PROFILE_URL}" +POWERSHELL_PROFILE_URL="${POWERSHELLPROFILEURL}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" POWERSHELL_ARCHIVE_ARCHITECTURES="amd64" @@ -165,26 +163,13 @@ if [ ${#POWERSHELL_MODULES[@]} -gt 0 ]; then done fi - # Determine the appropriate non-root user - if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then - USERNAME="" - POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") - for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do - if id -u ${CURRENT_USER} > /dev/null 2>&1; then - USERNAME=${CURRENT_USER} - break - fi - done - if [ "${USERNAME}" = "" ]; then - USERNAME=root - fi - elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then - USERNAME=root - fi + # If URL for powershell profile is provided, download it to '/opt/microsoft/powershell/7/profile.ps1' if [ -n "$POWERSHELL_PROFILE_URL" ]; then echo "Downloading PowerShell Profile from: $POWERSHELL_PROFILE_URL" - su ${USERNAME} -c "curl -sSL -o '/opt/microsoft/powershell/7/profile.ps1' '$POWERSHELL_PROFILE_URL'" + # Get profile path from currently installed pwsh + profilePath=$(pwsh -noni -c '$PROFILE.AllUsersAllHosts') + sudo -E curl -sSL -o "$profilePath" "$POWERSHELL_PROFILE_URL" fi # Clean up From d9e12f33552f81f52218e1701f69f00e86467bd5 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Mon, 4 Mar 2024 16:18:49 -0800 Subject: [PATCH 014/247] Automated documentation update (#893) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/powershell/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/powershell/README.md b/src/powershell/README.md index a44614604..f018778c0 100644 --- a/src/powershell/README.md +++ b/src/powershell/README.md @@ -17,7 +17,7 @@ Installs PowerShell along with needed dependencies. Useful for base Dockerfiles |-----|-----|-----|-----| | version | Select or enter a version of PowerShell. | string | latest | | modules | Optional comma separated list of PowerShell modules to install. | string | - | -| powershellProfileURL | Optional (publicly accessible) URL to download PowerShell profile. | string | - | +| powershellProfileURL | Optional (publicly accessible) URL to download PowerShell profile. | string | - | ## Customizations From 58ab1a1a6d3bad632c147447b1a9107183c8d43a Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 5 Mar 2024 11:25:53 -0800 Subject: [PATCH 015/247] [Git-Lfs] - Install previous version if current tags are missing artifacts (#894) * [Git-Lfs] - Install previous version if current tags are missing artifacts * fix test * fix debian:23 test --- src/git-lfs/devcontainer-feature.json | 7 ++++- src/git-lfs/install.sh | 37 +++++++++++++++++++++------ test/git-lfs/scenarios.json | 10 ++++++++ test/git-lfs/use_github.sh | 10 ++++++++ 4 files changed, 55 insertions(+), 9 deletions(-) create mode 100644 test/git-lfs/use_github.sh diff --git a/src/git-lfs/devcontainer-feature.json b/src/git-lfs/devcontainer-feature.json index 998b0b50e..ccdd99950 100644 --- a/src/git-lfs/devcontainer-feature.json +++ b/src/git-lfs/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "git-lfs", - "version": "1.1.1", + "version": "1.2.0", "name": "Git Large File Support (LFS)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/git-lfs", "description": "Installs Git Large File Support (Git LFS) along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like git and curl.", @@ -18,6 +18,11 @@ "type": "boolean", "default": true, "description": "Automatically pull LFS files when creating the container. When false, running 'git lfs pull' in the container will have the same effect." + }, + "installDirectlyFromGitHubRelease": { + "type": "boolean", + "default": false, + "description": "Installs 'git-lfs' from GitHub releases instead of package manager feeds" } }, "postCreateCommand": "/usr/local/share/pull-git-lfs-artifacts.sh", diff --git a/src/git-lfs/install.sh b/src/git-lfs/install.sh index 3f9aa2fff..2259cef59 100755 --- a/src/git-lfs/install.sh +++ b/src/git-lfs/install.sh @@ -9,6 +9,7 @@ GIT_LFS_VERSION=${VERSION:-"latest"} AUTO_PULL=${AUTOPULL:="true"} +INSTALL_WITH_GITHUB=${INSTALLDIRECTLYFROMGITHUBRELEASE:="false"} GIT_LFS_ARCHIVE_GPG_KEY_URI="https://packagecloud.io/github/git-lfs/gpgkey" GIT_LFS_ARCHIVE_ARCHITECTURES="amd64 arm64" @@ -130,14 +131,34 @@ install_using_apt() { git-lfs install --skip-repo } +# Function to fetch the version released prior to the latest version +get_previous_version() { + repo_url=$1 + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' +} + +install_from_release() { + git_lfs_filename="git-lfs-linux-${architecture}-v${GIT_LFS_VERSION}.tar.gz" + echo "Looking for release artfact: ${git_lfs_filename}" + curl -sSL -o "${git_lfs_filename}" "https://github.com/git-lfs/git-lfs/releases/download/v${GIT_LFS_VERSION}/${git_lfs_filename}" +} + install_using_github() { echo "(*) No apt package for ${VERSION_CODENAME} ${architecture}. Installing manually." mkdir -p /tmp/git-lfs cd /tmp/git-lfs find_version_from_git_tags GIT_LFS_VERSION "https://github.com/git-lfs/git-lfs" - git_lfs_filename="git-lfs-linux-${architecture}-v${GIT_LFS_VERSION}.tar.gz" - echo "Looking for release artfact: ${git_lfs_filename}" - curl -sSL -o "${git_lfs_filename}" "https://github.com/git-lfs/git-lfs/releases/download/v${GIT_LFS_VERSION}/${git_lfs_filename}" + install_from_release + + if grep -q "Not Found" "${git_lfs_filename}"; then + echo -e "\n(!) Failed to fetch the latest artifacts for Git lfs v${GIT_LFS_VERSION}..." + repo_url=https://api.github.com/repos/git-lfs/git-lfs/releases + requested_version=$(get_previous_version "${repo_url}") + echo -e "\nAttempting to install ${requested_version}" + GIT_LFS_VERSION=${requested_version#v} + install_from_release + fi + # Verify file curl -sSL -o "sha256sums.asc" "https://github.com/git-lfs/git-lfs/releases/download/v${GIT_LFS_VERSION}/sha256sums.asc" receive_gpg_keys GIT_LFS_CHECKSUM_GPG_KEYS @@ -165,7 +186,7 @@ export DEBIAN_FRONTEND=noninteractive # Install git, curl, gpg, dirmngr and debian-archive-keyring if missing . /etc/os-release -check_packages curl ca-certificates gnupg2 dirmngr apt-transport-https +check_packages curl ca-certificates gnupg2 dirmngr apt-transport-https jq if ! type git > /dev/null 2>&1; then check_packages git fi @@ -176,14 +197,14 @@ fi # Install Git LFS echo "Installing Git LFS..." architecture="$(dpkg --print-architecture)" -if [[ "${GIT_LFS_ARCHIVE_ARCHITECTURES}" = *"${architecture}"* ]] && [[ "${GIT_LFS_ARCHIVE_VERSION_CODENAMES}" = *"${VERSION_CODENAME}"* ]]; then - install_using_apt || use_github="true" +if [[ "${GIT_LFS_ARCHIVE_ARCHITECTURES}" = *"${architecture}"* ]] && [[ "${GIT_LFS_ARCHIVE_VERSION_CODENAMES}" = *"${VERSION_CODENAME}"* ]] && [[ "${INSTALL_WITH_GITHUB}" = "false" ]]; then + install_using_apt || INSTALL_WITH_GITHUB="true" else - use_github="true" + INSTALL_WITH_GITHUB="true" fi # If no archive exists or apt install fails, try direct from github -if [ "${use_github}" = "true" ]; then +if [ "${INSTALL_WITH_GITHUB}" = "true" ]; then install_using_github fi diff --git a/test/git-lfs/scenarios.json b/test/git-lfs/scenarios.json index 6cc3c24a5..009dfbb64 100644 --- a/test/git-lfs/scenarios.json +++ b/test/git-lfs/scenarios.json @@ -18,5 +18,15 @@ "autoPull": false } } + }, + "use_github": { + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", + "remoteUser": "vscode", + "features": { + "git-lfs": { + "version": "latest", + "installDirectlyFromGitHubRelease": true + } + } } } \ No newline at end of file diff --git a/test/git-lfs/use_github.sh b/test/git-lfs/use_github.sh new file mode 100644 index 000000000..458066f77 --- /dev/null +++ b/test/git-lfs/use_github.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "git-lfs" bash -c "git-lfs --version" + +reportResults \ No newline at end of file From 3c3a270a3703bb2b24857399f5e1d65120cb9152 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Tue, 5 Mar 2024 11:33:50 -0800 Subject: [PATCH 016/247] Automated documentation update (#895) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/git-lfs/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/git-lfs/README.md b/src/git-lfs/README.md index 9a9a06659..07f671755 100644 --- a/src/git-lfs/README.md +++ b/src/git-lfs/README.md @@ -17,6 +17,7 @@ Installs Git Large File Support (Git LFS) along with needed dependencies. Useful |-----|-----|-----|-----| | version | Select version of Git LFS to install | string | latest | | autoPull | Automatically pull LFS files when creating the container. When false, running 'git lfs pull' in the container will have the same effect. | boolean | true | +| installDirectlyFromGitHubRelease | Installs 'git-lfs' from GitHub releases instead of package manager feeds | boolean | false | From 965e12010f31baea61c7a9389082b19cfe498207 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Tue, 5 Mar 2024 17:20:49 -0800 Subject: [PATCH 017/247] [docker] - Install previous version if current tags are missing artifacts (#897) [docker-in-docker] - Install previous version if current tags are missing artifacts --- .../devcontainer-feature.json | 2 +- src/docker-in-docker/install.sh | 31 ++++++++++++++----- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index ab26a5e11..671f4e2ed 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.10.0", + "version": "2.10.1", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index 2a27f5c9c..dbe34e2ea 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -109,6 +109,20 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Function to fetch the version released prior to the latest version +get_previous_version() { + repo_url=$1 + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' +} + +install_compose_switch_fallback() { + echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." + previous_version=$(get_previous_version "https://api.github.com/repos/docker/compose-switch/releases") + echo -e "\nAttempting to install ${previous_version}" + compose_switch_version=${previous_version#v} + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch +} + ########################################### # Start docker-in-docker installation ########################################### @@ -290,6 +304,15 @@ if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then find_version_from_git_tags compose_version "https://github.com/docker/compose" "tags/v" echo "(*) Installing docker-compose ${compose_version}..." curl -L "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} + + if grep -q "Not Found" "${docker_compose_path}"; then + echo -e "\n(!) Failed to fetch the latest artifacts for docker-compose v${compose_version}..." + previous_version=$(get_previous_version "https://api.github.com/repos/docker/compose/releases") + echo -e "\nAttempting to install ${previous_version}" + compose_version=${previous_version#v} + curl -L "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} + fi + chmod +x ${docker_compose_path} # Download the SHA256 checksum @@ -310,7 +333,7 @@ if [ "${INSTALL_DOCKER_COMPOSE_SWITCH}" = "true" ] && ! type compose-switch > /d target_compose_path="$(dirname "${current_compose_path}")/docker-compose-v1" compose_switch_version="latest" find_version_from_git_tags compose_switch_version "https://github.com/docker/compose-switch" - curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch || install_compose_switch_fallback chmod +x /usr/local/bin/compose-switch # TODO: Verify checksum once available: https://github.com/docker/compose-switch/issues/11 # Setup v1 CLI as alternative in addition to compose-switch (which maps to v2) @@ -337,12 +360,6 @@ fi usermod -aG docker ${USERNAME} -# Function to fetch the version released prior to the latest version -get_previous_version() { - repo_url=$1 - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects -} - install_previous_version_artifacts() { wget_exit_code=$? if [ $wget_exit_code -eq 8 ]; then # failure due to 404: Not Found. From 8af4ce3dea3cdc2ea96062a47b3363b59aa9f1a2 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Mon, 11 Mar 2024 22:44:02 +0530 Subject: [PATCH 018/247] [docker-outside-of-docker] compose-switch can fallback to previous version (#901) * compose-switch fallback previous version * changes required --- .../devcontainer-feature.json | 2 +- src/docker-outside-of-docker/install.sh | 18 +++++++++- .../docker_build_compose_fallback.sh | 36 +++++++++++++++++++ test/docker-outside-of-docker/scenarios.json | 10 ++++++ 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 test/docker-outside-of-docker/docker_build_compose_fallback.sh diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index d4c1447ba..54fca5ec1 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.4.2", + "version": "1.4.3", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index 65424740e..dd39e1f68 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -99,6 +99,22 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Function to fetch the previous version of the plugin +get_previous_version() { + repo_url=$1 + # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' +} + + +install_compose_switch_fallback() { + echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." + previous_version=$(get_previous_version "https://api.github.com/repos/docker/compose-switch/releases") + echo -e "\nAttempting to install ${previous_version}" + compose_switch_version=${previous_version#v} + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose +} + # Ensure apt is in non-interactive to avoid prompts export DEBIAN_FRONTEND=noninteractive @@ -255,7 +271,7 @@ if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then echo "(*) Installing compose-switch as docker-compose..." compose_switch_version="latest" find_version_from_git_tags compose_switch_version "https://github.com/docker/compose-switch" - curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback chmod +x /usr/local/bin/docker-compose # TODO: Verify checksum once available: https://github.com/docker/compose-switch/issues/11 fi diff --git a/test/docker-outside-of-docker/docker_build_compose_fallback.sh b/test/docker-outside-of-docker/docker_build_compose_fallback.sh new file mode 100644 index 000000000..e1c0885e6 --- /dev/null +++ b/test/docker-outside-of-docker/docker_build_compose_fallback.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Optional: Import test library +source dev-container-features-test-lib + +check "installs compose-switch as docker-compose" bash -c "[[ -f /usr/local/bin/docker-compose ]]" + +# Fetch host/container arch. +architecture="$(dpkg --print-architecture)" + +repo_url="https://api.github.com/repos/docker/compose-switch/releases" + +# Function to fetch the previous version of the plugin +get_previous_version() { + sudo curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects +} + +install_compose_switch_fallback() { + echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch ${test_compose_switch_version}..." + previous_version=$(get_previous_version) + echo -e "\nAttempting to install ${previous_version}" + compose_switch_version=${previous_version} + sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose +} + +install_compose-switch_as_docker-compose() { + echo "(*) Installing compose-switch as docker-compose..." + test_compose_switch_version="1.2.xyz" + echo -e "\nTesting with $test_compose_switch_version..." + sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/${test_compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback + sudo chmod +x /usr/local/bin/docker-compose +} + +install_compose-switch_as_docker-compose + +check "installs compose-switch as docker-compose" bash -c "[[ -f /usr/local/bin/docker-compose ]]" \ No newline at end of file diff --git a/test/docker-outside-of-docker/scenarios.json b/test/docker-outside-of-docker/scenarios.json index 61f1ab402..6239a1ada 100644 --- a/test/docker-outside-of-docker/scenarios.json +++ b/test/docker-outside-of-docker/scenarios.json @@ -1,4 +1,14 @@ { + "docker_build_compose_fallback": { + "image": "mcr.microsoft.com/devcontainers/base:ubuntu-20.04", + "features": { + "docker-outside-of-docker": { + "moby": false, + "dockerDashComposeVersion": "latest" + } + }, + "containerUser": "vscode" + }, "docker_init_moby": { "image": "mcr.microsoft.com/devcontainers/base:ubuntu-20.04", "features": { From 460684dd971cb341b65d46fa2bd445f352bf75a2 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 13 Mar 2024 02:34:13 +0530 Subject: [PATCH 019/247] [Node] - nvm - fallback to previous version - code fix (#904) * [node] - nvm - added fallback to prev. version - if latest version tag released but binary not found * bumped patch version --- src/node/devcontainer-feature.json | 2 +- src/node/install.sh | 7 +++++-- test/node/nvm_test_fallback.sh | 27 +++++++++++++++++++++++++++ test/node/scenarios.json | 8 ++++++++ 4 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 test/node/nvm_test_fallback.sh diff --git a/src/node/devcontainer-feature.json b/src/node/devcontainer-feature.json index a93f75ace..f15ae4a4d 100644 --- a/src/node/devcontainer-feature.json +++ b/src/node/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "node", - "version": "1.4.0", + "version": "1.4.1", "name": "Node.js (via nvm), yarn and pnpm", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/node", "description": "Installs Node.js, nvm, yarn, pnpm, and needed dependencies.", diff --git a/src/node/install.sh b/src/node/install.sh index 493166d84..11a123534 100755 --- a/src/node/install.sh +++ b/src/node/install.sh @@ -293,8 +293,11 @@ set -e umask 0002 # Do not update profile - we'll do this manually export PROFILE=/dev/null -curl -so- "https://raw.githubusercontent.com/nvm-sh/nvm/v${NVM_VERSION}/install.sh" | bash - +curl -so- "https://raw.githubusercontent.com/nvm-sh/nvm/v${NVM_VERSION}/install.sh" | bash || { + PREV_NVM_VERSION=$(curl -s https://api.github.com/repos/nvm-sh/nvm/releases/latest | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/') + curl -so- "https://raw.githubusercontent.com/nvm-sh/nvm/\${PREV_NVM_VERSION}/install.sh" | bash + NVM_VERSION="\${PREV_NVM_VERSION}" +} source "${NVM_DIR}/nvm.sh" if [ "${NODE_VERSION}" != "" ]; then nvm alias default "${NODE_VERSION}" diff --git a/test/node/nvm_test_fallback.sh b/test/node/nvm_test_fallback.sh new file mode 100644 index 000000000..68e4071ac --- /dev/null +++ b/test/node/nvm_test_fallback.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +set -e + +trap 'echo "Error occurred at line $LINENO"; exit 1' ERR +source /usr/local/share/nvm/nvm.sh +#check nvm version +echo -e "\nโœ… nvm version as installed by feature = v$(nvm --version)"; +NVM_DIR="/usr/local/share/nvm" +NODE_VERSION="lts" +FAKE_NVM_VERSION="1.2.XYZ" +curl -so- "https://raw.githubusercontent.com/nvm-sh/nvm/v${FAKE_NVM_VERSION}/install.sh" | bash || { + PREV_NVM_VERSION=$(curl -s https://api.github.com/repos/nvm-sh/nvm/releases/latest | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/') + curl -so- "https://raw.githubusercontent.com/nvm-sh/nvm/${PREV_NVM_VERSION}/install.sh" | bash + NVM_VERSION="${PREV_NVM_VERSION}" +} + +#check nvm version +echo -e "\nโœ… nvm version as installed by test = v$(nvm --version)"; + +# Report result +reportResults diff --git a/test/node/scenarios.json b/test/node/scenarios.json index 262fba8ea..954c2c2e9 100644 --- a/test/node/scenarios.json +++ b/test/node/scenarios.json @@ -1,4 +1,12 @@ { + "nvm_test_fallback": { + "image": "debian:11", + "features": { + "node": { + "version": "lts" + } + } + }, "install_additional_node": { "image": "debian:11", "features": { From a674273c8602df49e03d3f74d7a0ccd5eea07cea Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 13 Mar 2024 02:36:34 +0530 Subject: [PATCH 020/247] [Powershell] - Fallback to prev. version - fix (#906) * [Powershell] - Fallback to prev. version - fix * patch version updated --- src/powershell/devcontainer-feature.json | 2 +- src/powershell/install.sh | 33 ++++++- .../install_powershell_fallback_test.sh | 97 +++++++++++++++++++ test/powershell/scenarios.json | 9 ++ 4 files changed, 135 insertions(+), 6 deletions(-) create mode 100644 test/powershell/install_powershell_fallback_test.sh diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index 86314be64..dd7a96b90 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.3.3", + "version": "1.3.4", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/powershell/install.sh b/src/powershell/install.sh index 0ce63c95d..4ba8bcf5c 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -106,6 +106,29 @@ install_using_apt() { apt-get install -yq powershell${version_suffix} || return 1 } +# Function to fetch the version released prior to the latest version +get_previous_version() { + repo_url=$1 + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' +} + +install_prev_pwsh() { + echo -e "\n(!) Failed to fetch the latest artifacts for powershell v${POWERSHELL_VERSION}..." + previous_version=$(get_previous_version "https://api.github.com/repos/PowerShell/PowerShell/releases") + echo -e "\nAttempting to install ${previous_version}" + POWERSHELL_VERSION="${previous_version#v}" + install_pwsh "${POWERSHELL_VERSION}" +} + +install_pwsh() { + POWERSHELL_VERSION=$1 + powershell_filename="powershell-${POWERSHELL_VERSION}-linux-${architecture}.tar.gz" + powershell_target_path="/opt/microsoft/powershell/$(echo ${POWERSHELL_VERSION} | grep -oE '[^\.]+' | head -n 1)" + mkdir -p /tmp/pwsh "${powershell_target_path}" + cd /tmp/pwsh + curl -sSL -o "${powershell_filename}" "https://github.com/PowerShell/PowerShell/releases/download/v${POWERSHELL_VERSION}/${powershell_filename}" +} + install_using_github() { # Fall back on direct download if no apt package exists in microsoft pool check_packages curl ca-certificates gnupg2 dirmngr libc6 libgcc1 libgssapi-krb5-2 libstdc++6 libunwind8 libuuid1 zlib1g libicu[0-9][0-9] @@ -116,11 +139,11 @@ install_using_github() { architecture="x64" fi find_version_from_git_tags POWERSHELL_VERSION https://github.com/PowerShell/PowerShell - powershell_filename="powershell-${POWERSHELL_VERSION}-linux-${architecture}.tar.gz" - powershell_target_path="/opt/microsoft/powershell/$(echo ${POWERSHELL_VERSION} | grep -oE '[^\.]+' | head -n 1)" - mkdir -p /tmp/pwsh "${powershell_target_path}" - cd /tmp/pwsh - curl -sSL -o "${powershell_filename}" "https://github.com/PowerShell/PowerShell/releases/download/v${POWERSHELL_VERSION}/${powershell_filename}" + install_pwsh "${POWERSHELL_VERSION}" + if grep -q "Not Found" "${powershell_filename}"; then + install_prev_pwsh + fi + # Ugly - but only way to get sha256 is to parse release HTML. Remove newlines and tags, then look for filename followed by 64 hex characters. curl -sSL -o "release.html" "https://github.com/PowerShell/PowerShell/releases/tag/v${POWERSHELL_VERSION}" powershell_archive_sha256="$(cat release.html | tr '\n' ' ' | sed 's|<[^>]*>||g' | grep -oP "${powershell_filename}\s+\K[0-9a-fA-F]{64}" || echo '')" diff --git a/test/powershell/install_powershell_fallback_test.sh b/test/powershell/install_powershell_fallback_test.sh new file mode 100644 index 000000000..852d7d103 --- /dev/null +++ b/test/powershell/install_powershell_fallback_test.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Extension-specific tests +check "az.resources" pwsh -Command "(Get-Module -ListAvailable -Name Az.Resources).Version.ToString()" +check "az.storage" pwsh -Command "(Get-Module -ListAvailable -Name Az.Storage).Version.ToString()" +check "profile" pwsh -Command "(Get-Variable $env:ProfileLoaded).Value" + +check "Powershell version as installed by feature" bash -c "pwsh --version" + +. /etc/os-release +architecture="$(dpkg --print-architecture)" + +get_previous_version() { + repo_url=$1 + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' +} + +install_prev_pwsh() { + echo -e "\n(!) Failed to fetch the latest artifacts for powershell v${POWERSHELL_VERSION}..." + previous_version=$(get_previous_version "https://api.github.com/repos/PowerShell/PowerShell/releases") + echo -e "\nAttempting to install ${previous_version}" + POWERSHELL_VERSION="${previous_version#v}" + install_pwsh "${POWERSHELL_VERSION}" +} + +install_pwsh() { + POWERSHELL_VERSION=$1 + powershell_filename="powershell-${POWERSHELL_VERSION}-linux-${architecture}.tar.gz" + powershell_target_path="/opt/microsoft/powershell/$(echo ${POWERSHELL_VERSION} | grep -oE '[^\.]+' | head -n 1)" + sudo mkdir -p /tmp/pwsh "${powershell_target_path}" + cd /tmp/pwsh + sudo curl -sSL -o "${powershell_filename}" "https://github.com/PowerShell/PowerShell/releases/download/v${POWERSHELL_VERSION}/${powershell_filename}" +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + sudo apt-get update -y + fi +} + +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + sudo chmod +x /var/lib/apt/lists/ + sudo mkdir -p /var/lib/apt/lists/partial + sudo chmod +rx /var/lib/dpkg/lock-frontend + apt_get_update + sudo apt-get -y install --no-install-recommends "$@" + fi +} + +install_using_github() { + # Fall back on direct download if no apt package exists in microsoft pool + check_packages curl ca-certificates gnupg2 dirmngr libc6 libgcc1 libgssapi-krb5-2 libstdc++6 libunwind8 libuuid1 zlib1g libicu[0-9][0-9] + if ! type git > /dev/null 2>&1; then + check_packages git + fi + if [ "${architecture}" = "amd64" ]; then + architecture="x64" + fi + + echo -e "\nTrying to install a non-existing version for Powershell..." + + POWERSHELL_VERSION="1.2.XYZ" + install_pwsh "${POWERSHELL_VERSION}" + + if grep -q "Not Found" "${powershell_filename}"; then + install_prev_pwsh + fi + + echo -e "\n" $POWERSHELL_VERSION "=powershell_version\n"; + # Ugly - but only way to get sha256 is to parse release HTML. Remove newlines and tags, then look for filename followed by 64 hex characters. + sudo curl -sSL -o "release.html" "https://github.com/PowerShell/PowerShell/releases/tag/v${POWERSHELL_VERSION}" + powershell_archive_sha256="$(cat release.html | tr '\n' ' ' | sed 's|<[^>]*>||g' | grep -oP "${powershell_filename}\s+\K[0-9a-fA-F]{64}" || echo '')" + if [ -z "${powershell_archive_sha256}" ]; then + echo "(!) WARNING: Failed to retrieve SHA256 for archive. Skipping validaiton." + else + echo "SHA256: ${powershell_archive_sha256}" + echo "${powershell_archive_sha256} *${powershell_filename}" | sha256sum -c - + fi + sudo tar xf "${powershell_filename}" -C "${powershell_target_path}" + sudo ln -s "${powershell_target_path}/pwsh" /usr/local/bin/pwsh + sudo rm -rf /tmp/pwsh +} + +install_using_github + +check "Powershell version as installed by test" bash -c "pwsh --version" + +# Report result +reportResults diff --git a/test/powershell/scenarios.json b/test/powershell/scenarios.json index b2659d7ad..6a810b0c9 100644 --- a/test/powershell/scenarios.json +++ b/test/powershell/scenarios.json @@ -7,5 +7,14 @@ "powershellProfileURL": "https://raw.githubusercontent.com/codspace/powershell-profile/main/Test-Profile.ps1" } } + }, + "install_powershell_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "powershell": { + "modules": "az.resources, az.storage", + "powershellProfileURL": "https://raw.githubusercontent.com/codspace/powershell-profile/main/Test-Profile.ps1" + } + } } } From 5ad3f6fbbf63096917622ac3da72266ad377c984 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 14 Mar 2024 03:59:50 +0530 Subject: [PATCH 021/247] [python] - cpython - prev. version fallback - fix (#907) * [python] - cpython - prev. version fallback - fix * according to comments --- src/python/devcontainer-feature.json | 2 +- src/python/install.sh | 94 ++++- ...tall_cpython_fallback_prev_version_test.sh | 355 ++++++++++++++++++ test/python/scenarios.json | 10 + 4 files changed, 441 insertions(+), 20 deletions(-) create mode 100644 test/python/install_cpython_fallback_prev_version_test.sh diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index c14dd7b97..9f211244f 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.4.1", + "version": "1.4.2", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.", diff --git a/src/python/install.sh b/src/python/install.sh index fcec73b0c..e8a9d24e1 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -248,6 +248,48 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + + # Use Oryx to install something using a partial version match oryx_install() { local platform=$1 @@ -368,6 +410,29 @@ install_openssl3() { rm -rf /tmp/openssl3 } +install_prev_vers_cpython() { + VERSION=$1 + echo -e "\n(!) Failed to fetch the latest artifacts for cpython ${VERSION}..." + find_prev_version_from_git_tags VERSION https://github.com/python/cpython + echo -e "\nAttempting to install ${VERSION}" + install_cpython "${VERSION}" +} + +install_cpython() { + VERSION=$1 + INSTALL_PATH="${PYTHON_INSTALL_PATH}/${VERSION}" + if [ -d "${INSTALL_PATH}" ]; then + echo "(!) Python version ${VERSION} already exists." + exit 1 + fi + mkdir -p /tmp/python-src ${INSTALL_PATH} + cd /tmp/python-src + cpython_tgz_filename="Python-${VERSION}.tgz" + cpython_tgz_url="https://www.python.org/ftp/python/${VERSION}/${cpython_tgz_filename}" + echo "Downloading ${cpython_tgz_filename}..." + curl -sSL -o "/tmp/python-src/${cpython_tgz_filename}" "${cpython_tgz_url}" +} + install_from_source() { VERSION=$1 echo "(*) Building Python ${VERSION} from source..." @@ -378,13 +443,6 @@ install_from_source() { # Find version using soft match find_version_from_git_tags VERSION "https://github.com/python/cpython" - INSTALL_PATH="${PYTHON_INSTALL_PATH}/${VERSION}" - - if [ -d "${INSTALL_PATH}" ]; then - echo "(!) Python version ${VERSION} already exists." - exit 1 - fi - # Some platforms/os versions need modern versions of openssl installed # via common package repositories, for now rhel-7 family, use case statement to # make it easy to expand @@ -396,23 +454,21 @@ install_from_source() { ;; esac - # Download tgz of source - mkdir -p /tmp/python-src ${INSTALL_PATH} - cd /tmp/python-src - local tgz_filename="Python-${VERSION}.tgz" - local tgz_url="https://www.python.org/ftp/python/${VERSION}/${tgz_filename}" - echo "Downloading ${tgz_filename}..." - curl -sSL -o "/tmp/python-src/${tgz_filename}" "${tgz_url}" - + install_cpython "${VERSION}" + if [ -f "/tmp/python-src/${cpython_tgz_filename}" ]; then + if grep -q "404 Not Found" "/tmp/python-src/${cpython_tgz_filename}"; then + install_prev_vers_cpython "${VERSION}" + fi + fi; # Verify signature if [[ ${VERSION_CODENAME} = "centos7" ]] || [[ ${VERSION_CODENAME} = "rhel7" ]]; then receive_gpg_keys_centos7 PYTHON_SOURCE_GPG_KEYS else receive_gpg_keys PYTHON_SOURCE_GPG_KEYS fi - echo "Downloading ${tgz_filename}.asc..." - curl -sSL -o "/tmp/python-src/${tgz_filename}.asc" "${tgz_url}.asc" - gpg --verify "${tgz_filename}.asc" + echo "Downloading ${cpython_tgz_filename}.asc..." + curl -sSL -o "/tmp/python-src/${cpython_tgz_filename}.asc" "${cpython_tgz_url}.asc" + gpg --verify "${cpython_tgz_filename}.asc" # Update min protocol for testing only - https://bugs.python.org/issue41561 if [ -f /etc/pki/tls/openssl.cnf ]; then @@ -424,7 +480,7 @@ install_from_source() { export OPENSSL_CONF=/tmp/python-src/openssl.cnf # Untar and build - tar -xzf "/tmp/python-src/${tgz_filename}" -C "/tmp/python-src" --strip-components=1 + tar -xzf "/tmp/python-src/${cpython_tgz_filename}" -C "/tmp/python-src" --strip-components=1 local config_args="" if [ "${OPTIMIZE_BUILD_FROM_SOURCE}" = "true" ]; then config_args="${config_args} --enable-optimizations" diff --git a/test/python/install_cpython_fallback_prev_version_test.sh b/test/python/install_cpython_fallback_prev_version_test.sh new file mode 100644 index 000000000..7056dba0c --- /dev/null +++ b/test/python/install_cpython_fallback_prev_version_test.sh @@ -0,0 +1,355 @@ +#!/bin/bash + +# Optional: Import test library +source dev-container-features-test-lib + +check "Python version as installed by Feature" bash -c "python3 -V" + +PYTHON_INSTALL_PATH="/usr/local/python" +OPTIMIZE_BUILD_FROM_SOURCE="false" +OVERRIDE_DEFAULT_VERSION="true" +ENABLESHARED="false" +if [ "$(id -u)" -ne 0 ]; then + echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' + exit 1 +fi +# Bring in ID, ID_LIKE, VERSION_ID, VERSION_CODENAME +. /etc/os-release +# Get an adjusted ID independent of distro variants +MAJOR_VERSION_ID=$(echo ${VERSION_ID} | cut -d . -f 1) +if [ "${ID}" = "debian" ] || [ "${ID_LIKE}" = "debian" ]; then + ADJUSTED_ID="debian" +elif [[ "${ID}" = "rhel" || "${ID}" = "fedora" || "${ID}" = "mariner" || "${ID_LIKE}" = *"rhel"* || "${ID_LIKE}" = *"fedora"* || "${ID_LIKE}" = *"mariner"* ]]; then + ADJUSTED_ID="rhel" + if [[ "${ID}" = "rhel" ]] || [[ "${ID}" = *"alma"* ]] || [[ "${ID}" = *"rocky"* ]]; then + VERSION_CODENAME="rhel${MAJOR_VERSION_ID}" + else + VERSION_CODENAME="${ID}${MAJOR_VERSION_ID}" + fi +else + echo "Linux distro ${ID} not supported." + exit 1 +fi + +# Setup INSTALL_CMD & PKG_MGR_CMD +if type apt-get > /dev/null 2>&1; then + PKG_MGR_CMD=apt-get + INSTALL_CMD="${PKG_MGR_CMD} -y install --no-install-recommends" +elif type microdnf > /dev/null 2>&1; then + PKG_MGR_CMD=microdnf + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" +elif type dnf > /dev/null 2>&1; then + PKG_MGR_CMD=dnf + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --refresh --best --nodocs --noplugins --setopt=install_weak_deps=0" +else + PKG_MGR_CMD=yum + INSTALL_CMD="${PKG_MGR_CMD} ${INSTALL_CMD_ADDL_REPOS} -y install --noplugins --setopt=install_weak_deps=0" +fi + +pkg_mgr_update() { + case $ADJUSTED_ID in + debian) + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + ${PKG_MGR_CMD} update -y + fi + ;; + rhel) + if [ ${PKG_MGR_CMD} = "microdnf" ]; then + if [ "$(ls /var/cache/yum/* 2>/dev/null | wc -l)" = 0 ]; then + echo "Running ${PKG_MGR_CMD} makecache ..." + ${PKG_MGR_CMD} makecache + fi + else + if [ "$(ls /var/cache/${PKG_MGR_CMD}/* 2>/dev/null | wc -l)" = 0 ]; then + echo "Running ${PKG_MGR_CMD} check-update ..." + set +e + ${PKG_MGR_CMD} check-update + rc=$? + if [ $rc != 0 ] && [ $rc != 100 ]; then + exit 1 + fi + set -e + fi + fi + ;; + esac +} + +check_packages() { + case ${ADJUSTED_ID} in + debian) + if ! dpkg -s "$@" > /dev/null 2>&1; then + pkg_mgr_update + ${INSTALL_CMD} "$@" + fi + ;; + rhel) + if ! rpm -q "$@" > /dev/null 2>&1; then + pkg_mgr_update + ${INSTALL_CMD} "$@" + fi + ;; + esac +} + +# Import the specified key in a variable name passed in as +receive_gpg_keys() { + local keys=${!1} + local keyring_args="" + local gpg_cmd="gpg" + if [ ! -z "$2" ]; then + mkdir -p "$(dirname \"$2\")" + keyring_args="--no-default-keyring --keyring $2" + fi + if [ ! -z "${KEYSERVER_PROXY}" ]; then + keyring_args="${keyring_args} --keyserver-options http-proxy=${KEYSERVER_PROXY}" + fi + + # Use a temporary location for gpg keys to avoid polluting image + export GNUPGHOME="/tmp/tmp-gnupg" + mkdir -p ${GNUPGHOME} + chmod 700 ${GNUPGHOME} + echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" > ${GNUPGHOME}/dirmngr.conf + # GPG key download sometimes fails for some reason and retrying fixes it. + local retry_count=0 + local gpg_ok="false" + set +e + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; + do + echo "(*) Downloading GPG key..." + ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" + if [ "${gpg_ok}" != "true" ]; then + echo "(*) Failed getting key, retring in 10s..." + (( retry_count++ )) + sleep 10s + fi + done + set -e + if [ "${gpg_ok}" = "false" ]; then + echo "(!) Failed to get gpg key." + exit 1 + fi +} +# RHEL7/CentOS7 has an older gpg that does not have dirmngr +# Iterate through keyservers until we have all the keys downloaded +receive_gpg_keys_centos7() { + local keys=${!1} + local keyring_args="" + local gpg_cmd="gpg" + if [ ! -z "$2" ]; then + mkdir -p "$(dirname \"$2\")" + keyring_args="--no-default-keyring --keyring $2" + fi + if [ ! -z "${KEYSERVER_PROXY}" ]; then + keyring_args="${keyring_args} --keyserver-options http-proxy=${KEYSERVER_PROXY}" + fi + + # Use a temporary location for gpg keys to avoid polluting image + export GNUPGHOME="/tmp/tmp-gnupg" + mkdir -p ${GNUPGHOME} + chmod 700 ${GNUPGHOME} + # GPG key download sometimes fails for some reason and retrying fixes it. + local retry_count=0 + local gpg_ok="false" + num_keys=$(echo ${keys} | wc -w) + set +e + echo "(*) Downloading GPG keys..." + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; do + for keyserver in $(echo "${GPG_KEY_SERVERS}" | sed 's/keyserver //'); do + ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys --keyserver=${keyserver} ) 2>&1 + downloaded_keys=$(gpg --list-keys | grep ^pub | wc -l) + if [[ ${num_keys} = ${downloaded_keys} ]]; then + gpg_ok="true" + break + fi + done + if [ "${gpg_ok}" != "true" ]; then + echo "(*) Failed getting key, retring in 10s..." + (( retry_count++ )) + sleep 10s + fi + done + set -e + if [ "${gpg_ok}" = "false" ]; then + echo "(!) Failed to get gpg key." + exit 1 + fi +} + + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +add_symlink() { + CURRENT_PATH="${PYTHON_INSTALL_PATH}/current" + if [[ ! -d "${CURRENT_PATH}" ]]; then + ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" + fi + + if [ "${OVERRIDE_DEFAULT_VERSION}" = "true" ]; then + if [[ $(ls -l ${CURRENT_PATH}) != *"-> ${INSTALL_PATH}"* ]] ; then + rm "${CURRENT_PATH}" + ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" + fi + fi +} + +install_prev_vers_cpython() { + VERSION=$1 + echo -e "\n(!) Failed to fetch the latest artifacts for cpython ${VERSION}..." + find_prev_version_from_git_tags VERSION https://github.com/python/cpython + echo -e "\nAttempting to install ${VERSION}" + install_cpython "${VERSION}" +} + +install_cpython() { + VERSION=$1 + INSTALL_PATH="${PYTHON_INSTALL_PATH}/${VERSION}" + mkdir -p /tmp/python-src ${INSTALL_PATH} + cd /tmp/python-src + cpython_tgz_filename="Python-${VERSION}.tgz" + cpython_tgz_url="https://www.python.org/ftp/python/${VERSION}/${cpython_tgz_filename}" + echo "Downloading ${cpython_tgz_filename}..." + curl -sSL -o "/tmp/python-src/${cpython_tgz_filename}" "${cpython_tgz_url}" +} + +install_from_source() { + VERSION=$1 + echo "(*) Building Python ${VERSION} from source..." + echo "(*) Building Python ${VERSION} from source..." + if ! type git > /dev/null 2>&1; then + check_packages git + fi + + echo -e "\nTrying to Install a fake version whose source binary wouldn't exist" + VERSION="3.12.xyz" + + install_cpython "${VERSION}" + if [ -f "/tmp/python-src/${cpython_tgz_filename}" ]; then + if grep -q "404 Not Found" "/tmp/python-src/${cpython_tgz_filename}"; then + # Use grep to search for "404 Not Found" in the file + echo "\"404 Not Found\" found in /tmp/python-src/${cpython_tgz_filename}. Not able to create source binary" + install_prev_vers_cpython "${VERSION}" + fi + fi; + + # Verify signature + if [[ ${VERSION_CODENAME} = "centos7" ]] || [[ ${VERSION_CODENAME} = "rhel7" ]]; then + receive_gpg_keys_centos7 PYTHON_SOURCE_GPG_KEYS + else + receive_gpg_keys PYTHON_SOURCE_GPG_KEYS + fi + + echo "Downloading ${cpython_tgz_filename}.asc..." + curl -sSL -o "/tmp/python-src/${cpython_tgz_filename}.asc" "${cpython_tgz_url}.asc" + + # Untar and build + tar -xzf "/tmp/python-src/${cpython_tgz_filename}" -C "/tmp/python-src" --strip-components=1 + local config_args="" + + if [ "${OPTIMIZE_BUILD_FROM_SOURCE}" = "true" ]; then + config_args="${config_args} --enable-optimizations" + fi + if [ "${ENABLESHARED}" = "true" ]; then + config_args=" ${config_args} --enable-shared" + # need double-$: LDFLAGS ends up in Makefile $$ becomes $ when evaluated. + # backslash needed for shell that Make calls escape the $. + export LDFLAGS="${LDFLAGS} -Wl,-rpath="'\$$ORIGIN'"/../lib" + fi + if [ -n "${ADDL_CONFIG_ARGS}" ]; then + config_args="${config_args} ${ADDL_CONFIG_ARGS}" + fi + + ./configure --prefix="${INSTALL_PATH}" --with-ensurepip=install ${config_args} + make -j 8 + make install + + cd /tmp + rm -rf /tmp/python-src ${GNUPGHOME} /tmp/vscdc-settings.env + + ln -s "${INSTALL_PATH}/bin/python3" "${INSTALL_PATH}/bin/python" + ln -s "${INSTALL_PATH}/bin/pip3" "${INSTALL_PATH}/bin/pip" + ln -s "${INSTALL_PATH}/bin/idle3" "${INSTALL_PATH}/bin/idle" + ln -s "${INSTALL_PATH}/bin/pydoc3" "${INSTALL_PATH}/bin/pydoc" + ln -s "${INSTALL_PATH}/bin/python3-config" "${INSTALL_PATH}/bin/python-config" + + add_symlink + +} + +install_from_source + +check "Python version as installed by Fallback Test" bash -c "python3 -V" \ No newline at end of file diff --git a/test/python/scenarios.json b/test/python/scenarios.json index 920e4f541..be37869df 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -1,4 +1,14 @@ { + "install_cpython_fallback_prev_version_test": { + "image": "python:3.12", + "features": { + "python": { + "version": "3.12", + "installTools": false, + "skipVulnerabilityPatching": true + } + } + }, "install_python310_skipVulnerabilityPatching_true": { "image": "python:3.10", "features": { From 2495879eade0b514aafc2ef253db93755883f552 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Fri, 15 Mar 2024 23:11:12 +0530 Subject: [PATCH 022/247] Reverting get_previous_version logic to get the first tag_name's value from json (#902) * changes as requested * changes acc. to comments --- src/kubectl-helm-minikube/devcontainer-feature.json | 2 +- src/kubectl-helm-minikube/install.sh | 4 ++-- test/docker-in-docker/docker_build_fallback_buildx.sh | 3 ++- test/kubectl-helm-minikube/install_only_helm_fallback.sh | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/kubectl-helm-minikube/devcontainer-feature.json b/src/kubectl-helm-minikube/devcontainer-feature.json index 16472205f..41a8b8b68 100644 --- a/src/kubectl-helm-minikube/devcontainer-feature.json +++ b/src/kubectl-helm-minikube/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "kubectl-helm-minikube", - "version": "1.1.7", + "version": "1.1.8", "name": "Kubectl, Helm, and Minikube", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/kubectl-helm-minikube", "description": "Installs latest version of kubectl, Helm, and optionally minikube. Auto-detects latest versions and installs needed dependencies.", diff --git a/src/kubectl-helm-minikube/install.sh b/src/kubectl-helm-minikube/install.sh index 69d4e0237..d7de3984f 100755 --- a/src/kubectl-helm-minikube/install.sh +++ b/src/kubectl-helm-minikube/install.sh @@ -158,8 +158,8 @@ fi # Function to fetch the version released prior to the latest version get_previous_version() { repo_url=$1 - # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' + # this would del the assets key and then get the first encountered tag_name's value from the filtered array of objects + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' } get_helm() { diff --git a/test/docker-in-docker/docker_build_fallback_buildx.sh b/test/docker-in-docker/docker_build_fallback_buildx.sh index 8d30d2a46..707e0a68a 100644 --- a/test/docker-in-docker/docker_build_fallback_buildx.sh +++ b/test/docker-in-docker/docker_build_fallback_buildx.sh @@ -26,7 +26,8 @@ get_latest_version() { # Function to fetch the previous version of the plugin get_previous_version() { - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects + # this would del the assets key and then get the first encountered tag_name's value from the filtered array of objects + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' } # Function to change the patch number in a semver version diff --git a/test/kubectl-helm-minikube/install_only_helm_fallback.sh b/test/kubectl-helm-minikube/install_only_helm_fallback.sh index 58ed137de..b8071a0b5 100644 --- a/test/kubectl-helm-minikube/install_only_helm_fallback.sh +++ b/test/kubectl-helm-minikube/install_only_helm_fallback.sh @@ -57,7 +57,7 @@ change_patch_number() { # Function to fetch the previous version of the plugin get_previous_version() { # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[1].tag_name' + curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' } get_helm() { From 5baf166f67683d6801c327d1c084d15f737947cd Mon Sep 17 00:00:00 2001 From: Leobaldo Alcantara Neto Date: Tue, 26 Mar 2024 14:05:35 -0300 Subject: [PATCH 023/247] Update DOCKER_DASH_COMPOSE_VERSION default to v2 (#920) * Update DOCKER_DASH_COMPOSE_VERSION default to v2 Changed it because devcontainer-feature.json documentation indicates that it's the default version. * Update devcontainer-feature.json version to 1.4.4 --- src/docker-outside-of-docker/devcontainer-feature.json | 2 +- src/docker-outside-of-docker/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index 54fca5ec1..2a55fdbca 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.4.3", + "version": "1.4.4", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index dd39e1f68..6b63469f9 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -10,7 +10,7 @@ DOCKER_VERSION="${VERSION:-"latest"}" USE_MOBY="${MOBY:-"true"}" MOBY_BUILDX_VERSION="${MOBYBUILDXVERSION:-"latest"}" -DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v1"}" # v1 or v2 or none +DOCKER_DASH_COMPOSE_VERSION="${DOCKERDASHCOMPOSEVERSION:-"v2"}" # v1 or v2 or none ENABLE_NONROOT_DOCKER="${ENABLE_NONROOT_DOCKER:-"true"}" SOURCE_SOCKET="${SOURCE_SOCKET:-"/var/run/docker-host.sock"}" From 0100f66138e2082cd76b92c30826cbb4544755a3 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 27 Mar 2024 05:25:37 +0530 Subject: [PATCH 024/247] [docker-in-docker]-docker_compose-fallback-github api (#914) * [docker-in-docker]-docker_compose-fallback-github api * changes misc. * misc. change * changes as required in review comments * changes miscellaneous * suggested changes * changes as requested by review comments on pr * Minor changes as suggested... --- .../devcontainer-feature.json | 2 +- src/docker-in-docker/install.sh | 142 +++++++++--- .../docker_build_fallback_buildx.sh | 206 +++++++++++++----- .../docker_build_fallback_compose.sh | 174 +++++++++++++++ test/docker-in-docker/scenarios.json | 13 +- 5 files changed, 442 insertions(+), 95 deletions(-) create mode 100644 test/docker-in-docker/docker_build_fallback_compose.sh diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index 671f4e2ed..812db444c 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.10.1", + "version": "2.10.2", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index dbe34e2ea..0dc9e52d1 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -109,18 +109,73 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + # Function to fetch the version released prior to the latest version get_previous_version() { - repo_url=$1 - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" } -install_compose_switch_fallback() { - echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." - previous_version=$(get_previous_version "https://api.github.com/repos/docker/compose-switch/releases") - echo -e "\nAttempting to install ${previous_version}" - compose_switch_version=${previous_version#v} - curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } ########################################### @@ -265,6 +320,16 @@ echo "Finished installing docker / moby!" docker_home="/usr/libexec/docker" cli_plugins_dir="${docker_home}/cli-plugins" +# fallback for docker-compose +fallback_compose(){ + local url=$1 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for docker-compose v${compose_version}..." + get_previous_version "${url}" "${repo_url}" compose_version + echo -e "\nAttempting to install v${compose_version}" + curl -fsSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} +} + # If 'docker-compose' command is to be included if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then case "${architecture}" in @@ -301,17 +366,16 @@ if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then fi else compose_version=${DOCKER_DASH_COMPOSE_VERSION#v} - find_version_from_git_tags compose_version "https://github.com/docker/compose" "tags/v" + docker_compose_url="https://github.com/docker/compose" + find_version_from_git_tags compose_version "$docker_compose_url" "tags/v" echo "(*) Installing docker-compose ${compose_version}..." - curl -L "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} - - if grep -q "Not Found" "${docker_compose_path}"; then - echo -e "\n(!) Failed to fetch the latest artifacts for docker-compose v${compose_version}..." - previous_version=$(get_previous_version "https://api.github.com/repos/docker/compose/releases") - echo -e "\nAttempting to install ${previous_version}" - compose_version=${previous_version#v} - curl -L "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} - fi + curl -fsSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} || { + if [[ $DOCKER_DASH_COMPOSE_VERSION == "latest" ]]; then + fallback_compose "$docker_compose_url" + else + echo -e "Error: Failed to install docker-compose v${compose_version}" + fi + } chmod +x ${docker_compose_path} @@ -325,6 +389,16 @@ if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then fi fi +# fallback method for compose-switch +fallback_compose-switch() { + local url=$1 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." + get_previous_version "$url" "$repo_url" compose_switch_version + echo -e "\nAttempting to install v${compose_switch_version}" + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch +} + # Install docker-compose switch if not already installed - https://github.com/docker/compose-switch#manual-installation if [ "${INSTALL_DOCKER_COMPOSE_SWITCH}" = "true" ] && ! type compose-switch > /dev/null 2>&1; then if type docker-compose > /dev/null 2>&1; then @@ -332,8 +406,9 @@ if [ "${INSTALL_DOCKER_COMPOSE_SWITCH}" = "true" ] && ! type compose-switch > /d current_compose_path="$(which docker-compose)" target_compose_path="$(dirname "${current_compose_path}")/docker-compose-v1" compose_switch_version="latest" - find_version_from_git_tags compose_switch_version "https://github.com/docker/compose-switch" - curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch || install_compose_switch_fallback + compose_switch_url="https://github.com/docker/compose-switch" + find_version_from_git_tags compose_switch_version "$compose_switch_url" + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/compose-switch || fallback_compose-switch "$compose_switch_url" chmod +x /usr/local/bin/compose-switch # TODO: Verify checksum once available: https://github.com/docker/compose-switch/issues/11 # Setup v1 CLI as alternative in addition to compose-switch (which maps to v2) @@ -360,29 +435,26 @@ fi usermod -aG docker ${USERNAME} -install_previous_version_artifacts() { - wget_exit_code=$? - if [ $wget_exit_code -eq 8 ]; then # failure due to 404: Not Found. - echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx v${buildx_version}..." - repo_url="https://api.github.com/repos/docker/buildx/releases" # GitHub repository URL - previous_version=$(get_previous_version "${repo_url}") - buildx_file_name="buildx-${previous_version}.linux-${architecture}" - echo -e "\nAttempting to install ${previous_version}" - wget https://github.com/docker/buildx/releases/download/${previous_version}/${buildx_file_name} - else - echo "(!) Failed to download docker buildx with exit code: $wget_exit_code" - exit 1 - fi +# fallback for docker/buildx +fallback_buildx() { + local url=$1 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx v${buildx_version}..." + get_previous_version "$url" "$repo_url" buildx_version + buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" + echo -e "\nAttempting to install v${buildx_version}" + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} } if [ "${INSTALL_DOCKER_BUILDX}" = "true" ]; then buildx_version="latest" - find_version_from_git_tags buildx_version "https://github.com/docker/buildx" "refs/tags/v" + docker_buildx_url="https://github.com/docker/buildx" + find_version_from_git_tags buildx_version "$docker_buildx_url" "refs/tags/v" echo "(*) Installing buildx ${buildx_version}..." buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" cd /tmp - wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} || install_previous_version_artifacts + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} || fallback_buildx "$docker_buildx_url" docker_home="/usr/libexec/docker" cli_plugins_dir="${docker_home}/cli-plugins" diff --git a/test/docker-in-docker/docker_build_fallback_buildx.sh b/test/docker-in-docker/docker_build_fallback_buildx.sh index 707e0a68a..e139613a0 100644 --- a/test/docker-in-docker/docker_build_fallback_buildx.sh +++ b/test/docker-in-docker/docker_build_fallback_buildx.sh @@ -14,83 +14,177 @@ check "docker-build" docker build ./ check "docker-buildx" bash -c "docker buildx version" check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" -echo -e "\n๐Ÿ‘‰${HL} Creating a scenario for fallback${N}\n" # Code to test the made up scenario when latest version of docker/buildx fails on wget command for fetching the artifacts -repo_url="https://api.github.com/repos/docker/buildx/releases" # GitHub repository URL architecture="$(dpkg --print-architecture)" +case "${architecture}" in + amd64) target_compose_arch=x86_64 ;; + arm64) target_compose_arch=aarch64 ;; + *) + echo "(!) Docker in docker does not support machine architecture '$architecture'. Please use an x86-64 or ARM64 machine." + exit 1 +esac -# Function to fetch the latest version of the plugin -get_latest_version() { - curl -s "$repo_url/latest" | jq -r '.tag_name' +docker_home="/usr/libexec/docker" +cli_plugins_dir="${docker_home}/cli-plugins" + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + err "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" } -# Function to fetch the previous version of the plugin -get_previous_version() { - # this would del the assets key and then get the first encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e } -# Function to change the patch number in a semver version -change_patch_number() { - local version="$1" # Input version - local new_patch="$2" # New patch number - # Extract major, minor, and current patch numbers - local major=$(echo "$version" | cut -d. -f1) - local minor=$(echo "$version" | cut -d. -f2) - local current_patch=$(echo "$version" | cut -d. -f3) - # Construct the new version with the updated patch number - local new_version="$major.$minor.$new_patch" - echo "$new_version" +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + echo -e "\nAttempting to find latest version using Github Api." + + output=$(curl -s "$repo_url"); + message=$(echo "$output" | jq -r '.message') + + if [[ $mode != "install_from_github_api_valid" ]]; then + message="API rate limit exceeded" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAttempting to find latest version using Github Api Failed. Exceeded API Rate Limit." + echo -e "\nAttempting to find latest version using Github Tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using Github Api Succeeded." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" } -change_version_to_fail() { - latest_version=$1 - new_patch_number="xyz" # for testing a tag not found scenario for docker/buildx plugin - latest_version=$(get_latest_version) # can take latest_version from fn get_latest_version - buildx_version_fallback_test=$(change_patch_number "$latest_version" "$new_patch_number") # for testing a tag not found scenario for docker/buildx plugin - echo "${buildx_version_fallback_test}" +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } -install_previous_version_artifacts() { - wget_exit_code=$? - if [ $wget_exit_code -ne 0 ]; then # means wget command to fetch latest version failed - if [ $wget_exit_code -eq 8 ]; then # failure due to 404: Not Found. - echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx ${buildx_version}..." - previous_version=$(get_previous_version) - echo -e "\nAttempting to install ${previous_version}" - buildx_file_name="buildx-${previous_version}.linux-${architecture}" - wget https://github.com/docker/buildx/releases/download/${previous_version}/${buildx_file_name} - else - echo "(!) Failed to download docker buildx with exit code: $wget_exit_code" - exit 1 - fi - fi +install_using_get_previous_version() { + local url=$1 + local mode=$2 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for docker buildx v${buildx_version}..." + get_previous_version "${url}" "${repo_url}" buildx_version "${mode}" + buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" + echo -e "\nAttempting to install v${buildx_version}" + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} } -test_version=$(change_version_to_fail "$(get_latest_version)") -buildx_file_name="buildx-${test_version}.linux-${architecture}" -buildx_version=$test_version +install_docker_buildx() { + mode=$1 + echo -e "\n${HL} Creating a scenario for fallback${N}\n" -# This wget command will fail as the wrong version won't fetch artifact -wget https://github.com/docker/buildx/releases/download/${buildx_version}/${buildx_file_name} || install_previous_version_artifacts + buildx_version="0.13.xyz" + echo "(*) Installing buildx ${buildx_version}..." + buildx_file_name="buildx-v${buildx_version}.linux-${architecture}" + cd /tmp -docker_home="/usr/libexec/docker" -cli_plugins_dir="${docker_home}/cli-plugins" + docker_buildx_url="https://github.com/docker/buildx" + wget https://github.com/docker/buildx/releases/download/v${buildx_version}/${buildx_file_name} || install_using_get_previous_version "${docker_buildx_url}" "${mode}" + + docker_home="/usr/libexec/docker" + cli_plugins_dir="${docker_home}/cli-plugins" -mkdir -p ${cli_plugins_dir} -mv ${buildx_file_name} ${cli_plugins_dir}/docker-buildx -chmod +x ${cli_plugins_dir}/docker-buildx + mkdir -p ${cli_plugins_dir} + mv ${buildx_file_name} ${cli_plugins_dir}/docker-buildx + chmod +x ${cli_plugins_dir}/docker-buildx -chown -R "${USERNAME}:docker" "${docker_home}" -chmod -R g+r+w "${docker_home}" -find "${docker_home}" -type d -print0 | xargs -n 1 -0 chmod g+s + chown -R "${USERNAME}:docker" "${docker_home}" + chmod -R g+r+w "${docker_home}" + find "${docker_home}" -type d -print0 | xargs -n 1 -0 chmod g+s +} + +echo -e "\n๐Ÿ‘‰${HL} docker-buildx version as installed by docker-in-docker test ( installing by github api ) ${N}" +install_docker_buildx "install_from_github_api_valid" + +# Definition specific tests after test for fallback +check "docker-buildx" docker buildx version +check "docker-buildx" bash -c "docker buildx version" + +echo -e "\n๐Ÿ‘‰${HL} docker-buildx version as installed by docker-in-docker test ( installing by find_prev_version_from_git_tags ) ${N}" +install_docker_buildx # Definition specific tests after test for fallback -echo -e "\n๐Ÿ‘‰${HL} docker/buildx version as installed by test for fallback${N}" check "docker-buildx" docker buildx version -check "docker-build" docker build ./ check "docker-buildx" bash -c "docker buildx version" -check "docker-buildx-path" bash -c "ls -la /usr/libexec/docker/cli-plugins/docker-buildx" # Report result reportResults diff --git a/test/docker-in-docker/docker_build_fallback_compose.sh b/test/docker-in-docker/docker_build_fallback_compose.sh new file mode 100644 index 000000000..4a18640f0 --- /dev/null +++ b/test/docker-in-docker/docker_build_fallback_compose.sh @@ -0,0 +1,174 @@ +#!/bin/bash + +# Optional: Import test library +source dev-container-features-test-lib + +# Setup STDERR. +err() { + echo "(!) $*" >&2 +} + +HL="\033[1;33m" +N="\033[0;37m" +echo -e "\n๐Ÿ‘‰${HL} docker-compose version as installed by docker-in-docker feature${N}" +check "docker-compose" bash -c "docker-compose version" + +architecture="$(dpkg --print-architecture)" +case "${architecture}" in + amd64) target_compose_arch=x86_64 ;; + arm64) target_compose_arch=aarch64 ;; + *) + echo "(!) Docker in docker does not support machine architecture '$architecture'. Please use an x86-64 or ARM64 machine." + exit 1 +esac + +docker_compose_path="/usr/local/bin/docker-compose" +cli_plugins_dir="${docker_home}/cli-plugins" + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + err "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + echo -e "\nAttempting to find latest version using Github Api." + + output=$(curl -s "$repo_url"); + message=$(echo "$output" | jq -r '.message') + + if [[ $mode != "install_from_github_api_valid" ]]; then + message="API rate limit exceeded" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAttempting to find latest version using Github Api Failed. Exceeded API Rate Limit." + echo -e "\nAttempting to find latest version using Github Tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using Github Api Succeeded." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_using_get_previous_version() { + local url=$1 + local mode=$2 + local repo_url=$(get_github_api_repo_url "$url") + echo -e "\n(!) Failed to fetch the latest artifacts for docker-compose v${compose_version}..." + get_previous_version "$url" "$repo_url" compose_version "$mode" + echo -e "\nAttempting to install v${compose_version}" + curl -fsSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} +} + +install_docker_compose() { + mode=$1 + compose_version="2.25.xyz" + docker_compose_url="https://github.com/docker/compose" + echo "(*) Installing docker-compose ${compose_version}..." + curl -fsSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}" -o ${docker_compose_path} || install_using_get_previous_version "$docker_compose_url" "$mode" +} + +chmod +x ${docker_compose_path} + +# Download the SHA256 checksum +DOCKER_COMPOSE_SHA256="$(curl -sSL "https://github.com/docker/compose/releases/download/v${compose_version}/docker-compose-linux-${target_compose_arch}.sha256" | awk '{print $1}')" +echo "${DOCKER_COMPOSE_SHA256} ${docker_compose_path}" > docker-compose.sha256sum +sha256sum -c docker-compose.sha256sum --ignore-missing + +mkdir -p ${cli_plugins_dir} +cp ${docker_compose_path} ${cli_plugins_dir} + +echo -e "\n๐Ÿ‘‰${HL} docker-compose version as installed by docker-in-docker test ( installing by github api ) ${N}" +install_docker_compose "install_from_github_api_valid" + +check "docker-compose" bash -c "docker-compose version" + +echo -e "\n๐Ÿ‘‰${HL} docker-compose version as installed by docker-in-docker test ( installing by find_prev_version_from_git_tags ) ${N}" +install_docker_compose + +check "docker-compose" bash -c "docker-compose version" diff --git a/test/docker-in-docker/scenarios.json b/test/docker-in-docker/scenarios.json index e209f3453..33333583d 100644 --- a/test/docker-in-docker/scenarios.json +++ b/test/docker-in-docker/scenarios.json @@ -1,4 +1,13 @@ { + "docker_build_fallback_compose": { + "image": "ubuntu:focal", + "features": { + "docker-in-docker": { + "version": "latest", + "dockerDashComposeVersion": "latest" + } + } + }, "dockerDefaultAddressPool": { "image": "mcr.microsoft.com/vscode/devcontainers/javascript-node:0-18", "remoteUser": "node", @@ -112,9 +121,7 @@ "features": { "docker-in-docker": { "version": "latest", - "installDockerBuildx": true, - "moby": "false", - "dockerDashComposeVersion": "v2" + "installDockerBuildx": true } } }, From 0a4fa18e1db577a60f569f7684dde62430a3179d Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 28 Mar 2024 03:30:20 +0530 Subject: [PATCH 025/247] [azure-cli] - python3.12 - feature broken due to distutils module removed - code fix (#909) * [azure-cli] - python3.12 - feature broken due to distutils module removed - fix * update default for installUsingPython * removed the installUsingPython=true flag from the building scenario * bump update the patch version --------- Co-authored-by: Samruddhi Khandale --- src/azure-cli/devcontainer-feature.json | 2 +- src/azure-cli/install.sh | 8 +++---- .../install_with_python_3_12_bookworm.sh | 21 +++++++++++++++++++ test/azure-cli/scenarios.json | 9 ++++++++ 4 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 test/azure-cli/install_with_python_3_12_bookworm.sh diff --git a/src/azure-cli/devcontainer-feature.json b/src/azure-cli/devcontainer-feature.json index 136b03fdc..f25e5f9a2 100644 --- a/src/azure-cli/devcontainer-feature.json +++ b/src/azure-cli/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "azure-cli", - "version": "1.2.2", + "version": "1.2.3", "name": "Azure CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/azure-cli", "description": "Installs the Azure CLI along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/azure-cli/install.sh b/src/azure-cli/install.sh index 6d3426699..7f4c4ff1c 100755 --- a/src/azure-cli/install.sh +++ b/src/azure-cli/install.sh @@ -15,10 +15,10 @@ rm -rf /var/lib/apt/lists/* AZ_VERSION=${VERSION:-"latest"} AZ_EXTENSIONS=${EXTENSIONS} AZ_INSTALLBICEP=${INSTALLBICEP:-false} -INSTALL_USING_PYTHON=${INSTALL_USING_PYTHON:-true} +INSTALL_USING_PYTHON=${INSTALL_USING_PYTHON:-false} MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" -AZCLI_ARCHIVE_ARCHITECTURES="amd64" -AZCLI_ARCHIVE_VERSION_CODENAMES="stretch buster bullseye bionic focal jammy" +AZCLI_ARCHIVE_ARCHITECTURES="amd64 arm64" +AZCLI_ARCHIVE_VERSION_CODENAMES="stretch bookworm buster bullseye bionic focal jammy" if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' @@ -183,7 +183,7 @@ install_with_complete_python_installation() { export DEBIAN_FRONTEND=noninteractive -# See if we're on x86_64 and if so, install via apt-get, otherwise use pip3 +# See if we're on x86_64 or AARCH64 and if so, install via apt-get, otherwise use pip3 echo "(*) Installing Azure CLI..." . /etc/os-release architecture="$(dpkg --print-architecture)" diff --git a/test/azure-cli/install_with_python_3_12_bookworm.sh b/test/azure-cli/install_with_python_3_12_bookworm.sh new file mode 100644 index 000000000..074876148 --- /dev/null +++ b/test/azure-cli/install_with_python_3_12_bookworm.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode +check "version" az --version + +echo -e "\n\n๐Ÿ”„ Testing 'O.S'" +if cat /etc/os-release | grep -q 'PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"'; then + echo -e "\n\nโœ… Passed 'O.S is Linux 12 (bookworm)'!" +else + echo -e "\n\nโŒ Failed 'O.S is other than Linux 12 (bookworm)'!" +fi + + +# Report result +reportResults \ No newline at end of file diff --git a/test/azure-cli/scenarios.json b/test/azure-cli/scenarios.json index b7a732395..041f50731 100644 --- a/test/azure-cli/scenarios.json +++ b/test/azure-cli/scenarios.json @@ -38,5 +38,14 @@ "installUsingPython": true } } + }, + "install_with_python_3_12_bookworm": { + "image": "mcr.microsoft.com/devcontainers/python:1-3.12-bookworm", + "user": "vscode", + "features": { + "azure-cli": { + "version": "latest" + } + } } } \ No newline at end of file From 988cdd2bb9eb9a96cf16ed30e2bfba32c68ad81c Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 28 Mar 2024 22:41:51 +0530 Subject: [PATCH 026/247] [Terraform] feature fallback fix (#912) * implementation of fallback logic for cosign, terraform * tflint, terragrunt - wrote fallback logic for these * misc change * [Terraform] - fallback - apply * changes as requested by review comments * misc change * misc change * 2-step fallback implemented in install.sh, test cases yet to be updated * install jq for ubuntu:focal, jammy & debian:11, 12 * test for terraform_docs updated * tfsec test updated * terraform fallback test updated * misc change * added tflint fallback test file * terragrunt fallback test file added * minor change in tfsec test file * few more changes to cosign, terragrunt installations.. * changes to test files for terragrunt & cosign * Changes acc. to review comments for pr * changes requested * changes as requested in review comments ! --- src/terraform/devcontainer-feature.json | 2 +- src/terraform/install.sh | 192 ++++++++++-- test/terraform/scenarios.json | 40 +++ .../terraform/terraform_docs_fallback_test.sh | 220 ++++++++++++++ test/terraform/terraform_fallback_test.sh | 206 +++++++++++++ test/terraform/terragrunt_fallback_test.sh | 219 +++++++++++++ test/terraform/tflint_fallback_test.sh | 287 ++++++++++++++++++ test/terraform/tfsec_fallback_test.sh | 224 ++++++++++++++ 8 files changed, 1372 insertions(+), 18 deletions(-) create mode 100644 test/terraform/terraform_docs_fallback_test.sh create mode 100644 test/terraform/terraform_fallback_test.sh create mode 100644 test/terraform/terragrunt_fallback_test.sh create mode 100644 test/terraform/tflint_fallback_test.sh create mode 100644 test/terraform/tfsec_fallback_test.sh diff --git a/src/terraform/devcontainer-feature.json b/src/terraform/devcontainer-feature.json index 51fb9a27a..5c9c042cc 100644 --- a/src/terraform/devcontainer-feature.json +++ b/src/terraform/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "terraform", - "version": "1.3.5", + "version": "1.3.6", "name": "Terraform, tflint, and TFGrunt", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/terraform", "description": "Installs the Terraform CLI and optionally TFLint and Terragrunt. Auto-detects latest version and installs needed dependencies.", diff --git a/src/terraform/install.sh b/src/terraform/install.sh index c1a382597..dc7aef8c3 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -137,6 +137,47 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + find_sentinel_version_from_url() { local variable_name=$1 local requested_version=${!variable_name} @@ -177,6 +218,73 @@ check_packages() { fi } +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + INSTALLER_FN=$3 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_cosign() { + COSIGN_VERSION=$1 + local URL=$2 + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + curl -L "${cosign_url}" -o $cosign_filename + if grep -q "Not Found" "$cosign_filename"; then + echo -e "\n(!) Failed to fetch the latest artifacts for cosign v${COSIGN_VERSION}..." + REPO_URL=$(get_github_api_repo_url "$URL") + get_previous_version "$URL" "$REPO_URL" COSIGN_VERSION + echo -e "\nAttempting to install ${COSIGN_VERSION}" + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + curl -L "${cosign_url}" -o $cosign_filename + fi + dpkg -i $cosign_filename + rm $cosign_filename + echo "Installation of cosign succeeded with ${COSIGN_VERSION}." +} + # Install 'cosign' for validating signatures # https://docs.sigstore.dev/cosign/overview/ ensure_cosign() { @@ -184,12 +292,10 @@ ensure_cosign() { if ! type cosign > /dev/null 2>&1; then echo "Installing cosign..." - LATEST_COSIGN_VERSION="latest" - find_version_from_git_tags LATEST_COSIGN_VERSION 'https://github.com/sigstore/cosign' - curl -L "https://github.com/sigstore/cosign/releases/latest/download/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb" -o /tmp/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb - - dpkg -i /tmp/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb - rm /tmp/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb + COSIGN_VERSION="latest" + cosign_url='https://github.com/sigstore/cosign' + find_version_from_git_tags COSIGN_VERSION "${cosign_url}" + install_cosign "${COSIGN_VERSION}" "${cosign_url}" fi if ! type cosign > /dev/null 2>&1; then echo "(!) Failed to install cosign." @@ -207,18 +313,30 @@ if ! type git > /dev/null 2>&1; then check_packages git fi +terraform_url='https://github.com/hashicorp/terraform' +tflint_url='https://github.com/terraform-linters/tflint' +terragrunt_url='https://github.com/gruntwork-io/terragrunt' # Verify requested version is available, convert latest -find_version_from_git_tags TERRAFORM_VERSION 'https://github.com/hashicorp/terraform' -find_version_from_git_tags TFLINT_VERSION 'https://github.com/terraform-linters/tflint' -find_version_from_git_tags TERRAGRUNT_VERSION 'https://github.com/gruntwork-io/terragrunt' +find_version_from_git_tags TERRAFORM_VERSION "$terraform_url" +find_version_from_git_tags TFLINT_VERSION "$tflint_url" +find_version_from_git_tags TERRAGRUNT_VERSION "$terragrunt_url" + +install_terraform() { + local TERRAFORM_VERSION=$1 + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + curl -sSL -o ${terraform_filename} "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${terraform_filename}" +} mkdir -p /tmp/tf-downloads cd /tmp/tf-downloads - # Install Terraform, tflint, Terragrunt echo "Downloading terraform..." terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" -curl -sSL -o ${terraform_filename} "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${terraform_filename}" +install_terraform "$TERRAFORM_VERSION" +if grep -q "The specified key does not exist." "${terraform_filename}"; then + install_previous_version TERRAFORM_VERSION $terraform_url "install_terraform" + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" +fi if [ "${TERRAFORM_SHA256}" != "dev-mode" ]; then if [ "${TERRAFORM_SHA256}" = "automatic" ]; then receive_gpg_keys TERRAFORM_GPG_KEY @@ -233,10 +351,18 @@ fi unzip ${terraform_filename} mv -f terraform /usr/local/bin/ +install_tflint() { + TFLINT_VERSION=$1 + curl -sSL -o /tmp/tf-downloads/${TFLINT_FILENAME} https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/${TFLINT_FILENAME} +} + if [ "${TFLINT_VERSION}" != "none" ]; then echo "Downloading tflint..." TFLINT_FILENAME="tflint_linux_${architecture}.zip" - curl -sSL -o /tmp/tf-downloads/${TFLINT_FILENAME} https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/${TFLINT_FILENAME} + install_tflint "$TFLINT_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${TFLINT_FILENAME}"; then + install_previous_version TFLINT_VERSION "$tflint_url" "install_tflint" + fi if [ "${TFLINT_SHA256}" != "dev-mode" ]; then if [ "${TFLINT_SHA256}" != "automatic" ]; then @@ -277,10 +403,20 @@ if [ "${TFLINT_VERSION}" != "none" ]; then unzip /tmp/tf-downloads/${TFLINT_FILENAME} mv -f tflint /usr/local/bin/ fi + +install_terragrunt() { + TERRAGRUNT_VERSION=$1 + curl -sSL -o /tmp/tf-downloads/${terragrunt_filename} https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/${terragrunt_filename} +} + if [ "${TERRAGRUNT_VERSION}" != "none" ]; then echo "Downloading Terragrunt..." terragrunt_filename="terragrunt_linux_${architecture}" - curl -sSL -o /tmp/tf-downloads/${terragrunt_filename} https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/${terragrunt_filename} + install_terragrunt "$TERRAGRUNT_VERSION" + output=$(cat "/tmp/tf-downloads/${terragrunt_filename}") + if [[ $output == "Not Found" ]]; then + install_previous_version TERRAGRUNT_VERSION $terragrunt_url "install_terragrunt" + fi if [ "${TERRAGRUNT_SHA256}" != "dev-mode" ]; then if [ "${TERRAGRUNT_SHA256}" = "automatic" ]; then curl -sSL -o terragrunt_SHA256SUMS https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/SHA256SUMS @@ -318,12 +454,23 @@ if [ "${INSTALL_SENTINEL}" = "true" ]; then mv -f /tmp/tf-downloads/sentinel /usr/local/bin/sentinel fi +install_tfsec() { + local TFSEC_VERSION=$1 + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfsec_filename} https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${tfsec_filename} +} + if [ "${INSTALL_TFSEC}" = "true" ]; then TFSEC_VERSION="latest" - find_version_from_git_tags TFSEC_VERSION 'https://github.com/aquasecurity/tfsec' + tfsec_url='https://github.com/aquasecurity/tfsec' + find_version_from_git_tags TFSEC_VERSION $tfsec_url tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" echo "(*) Downloading TFSec... ${tfsec_filename}" - curl -sSL -o /tmp/tf-downloads/${tfsec_filename} https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${tfsec_filename} + install_tfsec "$TFSEC_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfsec_filename}"; then + install_previous_version TFSEC_VERSION $tfsec_url "install_tfsec" + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + fi if [ "${TFSEC_SHA256}" != "dev-mode" ]; then if [ "${TFSEC_SHA256}" = "automatic" ]; then curl -sSL -o tfsec_SHA256SUMS https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/tfsec_${TFSEC_VERSION}_checksums.txt @@ -338,12 +485,23 @@ if [ "${INSTALL_TFSEC}" = "true" ]; then mv -f /tmp/tf-downloads/tfsec/tfsec /usr/local/bin/tfsec fi +install_terraform_docs() { + local TERRAFORM_DOCS_VERSION=$1 + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfdocs_filename} https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/${tfdocs_filename} +} + if [ "${INSTALL_TERRAFORM_DOCS}" = "true" ]; then TERRAFORM_DOCS_VERSION="latest" - find_version_from_git_tags TERRAFORM_DOCS_VERSION 'https://github.com/terraform-docs/terraform-docs' + terraform_docs_url='https://github.com/terraform-docs/terraform-docs' + find_version_from_git_tags TERRAFORM_DOCS_VERSION $terraform_docs_url tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" echo "(*) Downloading Terraform docs... ${tfdocs_filename}" - curl -sSL -o /tmp/tf-downloads/${tfdocs_filename} https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/${tfdocs_filename} + install_terraform_docs "$TERRAFORM_DOCS_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfdocs_filename}"; then + install_previous_version TERRAFORM_DOCS_VERSION $terraform_docs_url "install_terraform_docs" + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + fi if [ "${TERRAFORM_DOCS_SHA256}" != "dev-mode" ]; then if [ "${TERRAFORM_DOCS_SHA256}" = "automatic" ]; then curl -sSL -o tfdocs_SHA256SUMS https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/terraform-docs-v${TERRAFORM_DOCS_VERSION}.sha256sum diff --git a/test/terraform/scenarios.json b/test/terraform/scenarios.json index 0a4d8956c..04693666c 100644 --- a/test/terraform/scenarios.json +++ b/test/terraform/scenarios.json @@ -15,6 +15,14 @@ } } }, + "tfsec_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "installTFsec": true + } + } + }, "install_terraform_docs": { "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { @@ -23,6 +31,38 @@ } } }, + "terraform_docs_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "installTerraformDocs": true + } + } + }, + "terraform_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "version": "latest" + } + } + }, + "terragrunt_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "terragrunt": "latest" + } + } + }, + "tflint_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "tflint": "latest" + } + } + }, "older_tflint": { "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { diff --git a/test/terraform/terraform_docs_fallback_test.sh b/test/terraform/terraform_docs_fallback_test.sh new file mode 100644 index 000000000..256ef2f85 --- /dev/null +++ b/test/terraform/terraform_docs_fallback_test.sh @@ -0,0 +1,220 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +# Terraform Docs specific tests +check "terraform-docs version as installed by feature" terraform-docs --version + +TERRAFORM_DOCS_SHA256="automatic" + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + INSTALLER_FN=$4 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_terraform_docs() { + local TERRAFORM_DOCS_VERSION=$1 + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfdocs_filename} https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/${tfdocs_filename} +} + + +try_install_terraform_docs_dummy_version() { + mode=$1 + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + TERRAFORM_DOCS_VERSION="0.17.xyz" + echo -e "\nInstalling TERRAFORM_DOCS dummy version.." v${TERRAFORM_DOCS_VERSION} + terraform_docs_url='https://github.com/terraform-docs/terraform-docs' + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + echo "(*) Downloading Terraform docs... ${tfdocs_filename}" + install_terraform_docs "$TERRAFORM_DOCS_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfdocs_filename}"; then + install_previous_version TERRAFORM_DOCS_VERSION $terraform_docs_url $mode "install_terraform_docs" + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + fi + if [ "${TERRAFORM_DOCS_SHA256}" != "dev-mode" ]; then + if [ "${TERRAFORM_DOCS_SHA256}" = "automatic" ]; then + curl -sSL -o tfdocs_SHA256SUMS https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/terraform-docs-v${TERRAFORM_DOCS_VERSION}.sha256sum + else + echo "${TERRAFORM_DOCS_SHA256} *${tfsec_filename}" > tfdocs_SHA256SUMS + fi + sha256sum --ignore-missing -c tfdocs_SHA256SUMS + fi + mkdir -p /tmp/tf-downloads/tfdocs + tar -xzf /tmp/tf-downloads/${tfdocs_filename} -C /tmp/tf-downloads/tfdocs + sudo chmod a+x /tmp/tf-downloads/tfdocs/terraform-docs + sudo mv -f /tmp/tf-downloads/tfdocs/terraform-docs /usr/local/bin/terraform-docs +} + +try_install_terraform_docs_dummy_version "mode1" + +check "terraform-docs version as installed by test (mode 1: install using find_prev_version_from_git_tags)" terraform-docs --version + +try_install_terraform_docs_dummy_version "mode2" + +check "terraform-docs version as installed by test (mode 2: install using GitHub Api)" terraform-docs --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/terraform/terraform_fallback_test.sh b/test/terraform/terraform_fallback_test.sh new file mode 100644 index 000000000..a5193daa4 --- /dev/null +++ b/test/terraform/terraform_fallback_test.sh @@ -0,0 +1,206 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +check "terraform version as installed by feature" terraform --version + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + INSTALLER_FN=$4 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_terraform() { + local TERRAFORM_VERSION=$1 + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + curl -sSL -o ${terraform_filename} "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${terraform_filename}" +} + +try_install_dummy_terraform_version() { + mode=$1 + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + terraform_url='https://github.com/hashicorp/terraform' + TERRAFORM_VERSION="1.7.xyz" + echo -e "\nAttempting to install dummy version for Terraform v${TERRAFORM_VERSION}..." + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + install_terraform "$TERRAFORM_VERSION" + if grep -q "The specified key does not exist." "${terraform_filename}"; then + install_previous_version TERRAFORM_VERSION $terraform_url $mode "install_terraform" + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + fi + unzip ${terraform_filename} + sudo mv -f terraform /usr/local/bin/ +} + +try_install_dummy_terraform_version "mode1" + +check "terraform version as installed by test after fallbacking from the dummy version (mode 1: install using find_prev_version_from_git_tags)" terraform --version + +try_install_dummy_terraform_version "mode2" + +check "terraform version as installed by test after fallbacking from the dummy version (mode 2: install using GitHub Api)" terraform --version + +# Report result +reportResults + diff --git a/test/terraform/terragrunt_fallback_test.sh b/test/terraform/terragrunt_fallback_test.sh new file mode 100644 index 000000000..83289380f --- /dev/null +++ b/test/terraform/terragrunt_fallback_test.sh @@ -0,0 +1,219 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +check "terragrunt version as installed by feature" terragrunt --version + +TERRAGRUNT_SHA256="automatic" + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + INSTALLER_FN=$4 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + + +install_terragrunt() { + TERRAGRUNT_VERSION=$1 + curl -sSL -o /tmp/tf-downloads/${terragrunt_filename} https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/${terragrunt_filename} +} + + +try_install_dummy_terragrunt_version() { + mode=$1 + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + terragrunt_url='https://github.com/gruntwork-io/terragrunt' + TERRAGRUNT_VERSION="0.55.xyz" + echo -e "\nAttempting to install terragrunt dummy v${TERRAGRUNT_VERSION}" + echo "Downloading Terragrunt... v${TERRAGRUNT_VERSION}" + terragrunt_filename="terragrunt_linux_${architecture}" + install_terragrunt "$TERRAGRUNT_VERSION" + output=$(cat "/tmp/tf-downloads/${terragrunt_filename}") + if [[ $output == "Not Found" ]]; then + install_previous_version TERRAGRUNT_VERSION $terragrunt_url $mode "install_terragrunt" + fi + if [ "${TERRAGRUNT_SHA256}" != "dev-mode" ]; then + if [ "${TERRAGRUNT_SHA256}" = "automatic" ]; then + curl -sSL -o terragrunt_SHA256SUMS https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/SHA256SUMS + else + echo "${TERRAGRUNT_SHA256} *${terragrunt_filename}" > terragrunt_SHA256SUMS + fi + sha256sum --ignore-missing -c terragrunt_SHA256SUMS + fi + sudo chmod a+x /tmp/tf-downloads/${terragrunt_filename} + sudo mv -f /tmp/tf-downloads/${terragrunt_filename} /usr/local/bin/terragrunt +} + +try_install_dummy_terragrunt_version "mode1" + +check "terragrunt version as installed by test after fallbacking from the dummy version (mode 1: install using find_prev_version_from_git_tags)" terragrunt --version + +try_install_dummy_terragrunt_version "mode2" + +check "terragrunt version as installed by test after fallbacking from the dummy version (mode 2: install using GitHub Api)" terragrunt --version + +# Report result +reportResults + diff --git a/test/terraform/tflint_fallback_test.sh b/test/terraform/tflint_fallback_test.sh new file mode 100644 index 000000000..5619ff4fb --- /dev/null +++ b/test/terraform/tflint_fallback_test.sh @@ -0,0 +1,287 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +TFLINT_SHA256="automatic" + +GPG_KEY_SERVERS="keyserver hkps://keyserver.ubuntu.com +keyserver hkps://keys.openpgp.org +keyserver hkps://keyserver.pgp.com" + +check "tflint version as installed by feature" tflint --version +check "cosign version as installed by feature" cosign version + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + INSTALLER_FN=$4 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_cosign() { + COSIGN_VERSION=$1 + local URL=$2 + local mode=$3 + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + curl -L "${cosign_url}" -o $cosign_filename + if grep -q "Not Found" "$cosign_filename"; then + echo -e "\n(!) Failed to fetch the latest artifacts for cosign v${COSIGN_VERSION}..." + REPO_URL=$(get_github_api_repo_url "$URL") + get_previous_version "$URL" "$REPO_URL" COSIGN_VERSION $mode + echo -e "\nAttempting to install ${COSIGN_VERSION}" + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + curl -L "${cosign_url}" -o $cosign_filename + fi + dpkg -i $cosign_filename + rm $cosign_filename + echo "Installation of cosign succeeded with ${COSIGN_VERSION}." +} + +# Install 'cosign' for validating signatures +# https://docs.sigstore.dev/cosign/overview/ +ensure_cosign() { + mode=$1 + if ! type cosign > /dev/null 2>&1; then + echo -e "\nAttempting to install dummy cosign version..." + COSIGN_VERSION="2.2.xyz" + echo "Installing cosign... v${COSIGN_VERSION}" + cosign_url='https://github.com/sigstore/cosign' + install_cosign "${COSIGN_VERSION}" "${cosign_url}" $mode + fi + if ! type cosign > /dev/null 2>&1; then + echo "(!) Failed to install cosign." + exit 1 + fi + cosign version +} + +install_tflint() { + TFLINT_VERSION=$1 + curl -sSL -o /tmp/tf-downloads/${TFLINT_FILENAME} https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/${TFLINT_FILENAME} +} + + +try_install_dummy_tflint_cosign_version() { + mode=$1 + tflint_url='https://github.com/terraform-linters/tflint' + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + echo -e "\nTrying to install dummy tflint version..." + TFLINT_VERSION="0.50.XYZ" + echo "Downloading tflint...v${TFLINT_VERSION}" + TFLINT_FILENAME="tflint_linux_${architecture}.zip" + install_tflint "$TFLINT_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${TFLINT_FILENAME}"; then + install_previous_version TFLINT_VERSION "$tflint_url" $mode "install_tflint" + fi + if [ "${TFLINT_SHA256}" != "dev-mode" ]; then + + if [ "${TFLINT_SHA256}" != "automatic" ]; then + echo "${TFLINT_SHA256} *${TFLINT_FILENAME}" > tflint_checksums.txt + sha256sum --ignore-missing -c tflint_checksums.txt + else + curl -sSL -o tflint_checksums.txt https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt + + set +e + curl -sSL -o checksums.txt.keyless.sig https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt.keyless.sig + set -e + + # Check that checksums.txt.keyless.sig exists and is not empty + if [ -s checksums.txt.keyless.sig ]; then + # Validate checksums with cosign + curl -sSL -o checksums.txt.pem https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt.pem + ensure_cosign $mode + cosign verify-blob \ + --certificate=/tmp/tf-downloads/checksums.txt.pem \ + --signature=/tmp/tf-downloads/checksums.txt.keyless.sig \ + --certificate-identity-regexp="^https://github.com/terraform-linters/tflint" \ + --certificate-oidc-issuer=https://token.actions.githubusercontent.com \ + /tmp/tf-downloads/tflint_checksums.txt + # Ensure that checksums.txt has $TFLINT_FILENAME + grep ${TFLINT_FILENAME} /tmp/tf-downloads/tflint_checksums.txt + # Validate downloaded file + sha256sum --ignore-missing -c tflint_checksums.txt + else + # Fallback to older, GPG-based verification (pre-0.47.0 of tflint) + curl -sSL -o tflint_checksums.txt.sig https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt.sig + curl -sSL -o tflint_key "${TFLINT_GPG_KEY_URI}" + gpg -q --import tflint_key + gpg --verify tflint_checksums.txt.sig tflint_checksums.txt + fi + fi + fi + + unzip /tmp/tf-downloads/${TFLINT_FILENAME} + sudo mv -f tflint /usr/local/bin/ +} + +try_install_dummy_tflint_cosign_version "mode1" + +check "tflint version as installed when mode=1" tflint --version +check "cosign version as installed when mode=1" cosign version + +try_install_dummy_tflint_cosign_version "mode2" + +check "tflint version as installed when mode=2" tflint --version +check "cosign version as installed when mode=2" cosign version \ No newline at end of file diff --git a/test/terraform/tfsec_fallback_test.sh b/test/terraform/tfsec_fallback_test.sh new file mode 100644 index 000000000..5c87d821e --- /dev/null +++ b/test/terraform/tfsec_fallback_test.sh @@ -0,0 +1,224 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +# Trap errors and call the error handling function +trap 'handle_error' ERR + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +TFSEC_SHA256="automatic" + +# TFSec specific tests +check "tfsec version as installed by feature" tfsec --version + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + # install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + INSTALLER_FN=$4 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_tfsec() { + local TFSEC_VERSION=$1 + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfsec_filename} https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${tfsec_filename} +} + +try_install_tfsec_dummy_version() { + mode=$1 + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + TFSEC_VERSION="1.28.XYZ" + echo -e "\nInstalling TFSEC dummy version.." v${TFSEC_VERSION} + tfsec_url='https://github.com/aquasecurity/tfsec' + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + echo "(*) Downloading TFSec... ${tfsec_filename}" + install_tfsec "$TFSEC_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfsec_filename}"; then + install_previous_version TFSEC_VERSION $tfsec_url $mode "install_tfsec" + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + fi + if [ "${TFSEC_SHA256}" != "dev-mode" ]; then + if [ "${TFSEC_SHA256}" = "automatic" ]; then + curl -sSL -o tfsec_SHA256SUMS https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/tfsec_${TFSEC_VERSION}_checksums.txt + else + echo "${TFSEC_SHA256} *${tfsec_filename}" > tfsec_SHA256SUMS + fi + sha256sum --ignore-missing -c tfsec_SHA256SUMS + fi + mkdir -p /tmp/tf-downloads/tfsec + tar -xzf /tmp/tf-downloads/${tfsec_filename} -C /tmp/tf-downloads/tfsec + chmod a+x /tmp/tf-downloads/tfsec/tfsec + sudo mv -f /tmp/tf-downloads/tfsec/tfsec /usr/local/bin/tfsec +} + +try_install_tfsec_dummy_version "mode1" + +check "tfsec version as installed by test after fallbacking from the dummy version (mode 1: install using find_prev_version_from_git_tags)" tfsec --version + +try_install_tfsec_dummy_version "mode2" + +check "tfsec version as installed by test after fallbacking from the dummy version (mode 2: install using GitHub Api)" tfsec --version + +# Report result +reportResults \ No newline at end of file From cf03551a529f39f4a50b37e39fac0964c7a579ca Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 28 Mar 2024 22:58:53 +0530 Subject: [PATCH 027/247] [kubectl-helm-minikube] - helm - alternative fallback method implemented (#923) * [kubectl-helm-minikube] - helm - alternative fallback method implemented * bump patch version --- .../devcontainer-feature.json | 2 +- src/kubectl-helm-minikube/install.sh | 80 +++++++- .../install_only_helm_fallback.sh | 192 ++++++++++++------ 3 files changed, 200 insertions(+), 74 deletions(-) diff --git a/src/kubectl-helm-minikube/devcontainer-feature.json b/src/kubectl-helm-minikube/devcontainer-feature.json index 41a8b8b68..8c5dfddc0 100644 --- a/src/kubectl-helm-minikube/devcontainer-feature.json +++ b/src/kubectl-helm-minikube/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "kubectl-helm-minikube", - "version": "1.1.8", + "version": "1.1.9", "name": "Kubectl, Helm, and Minikube", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/kubectl-helm-minikube", "description": "Installs latest version of kubectl, Helm, and optionally minikube. Auto-detects latest versions and installs needed dependencies.", diff --git a/src/kubectl-helm-minikube/install.sh b/src/kubectl-helm-minikube/install.sh index d7de3984f..871a77ca1 100755 --- a/src/kubectl-helm-minikube/install.sh +++ b/src/kubectl-helm-minikube/install.sh @@ -88,6 +88,47 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + apt_get_update() { if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then @@ -157,9 +198,32 @@ fi # Function to fetch the version released prior to the latest version get_previous_version() { - repo_url=$1 - # this would del the assets key and then get the first encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name#v} + + output=$(curl -s "$repo_url"); + + check_packages jq + + message=$(echo "$output" | jq -r '.message') + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="v${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } get_helm() { @@ -173,7 +237,8 @@ get_helm() { if [ ${HELM_VERSION} != "none" ]; then # Install Helm, verify signature and checksum echo "Downloading Helm..." - find_version_from_git_tags HELM_VERSION "https://github.com/helm/helm" + helm_url="https://github.com/helm/helm" + find_version_from_git_tags HELM_VERSION "${helm_url}" if [ "${HELM_VERSION::1}" != 'v' ]; then HELM_VERSION="v${HELM_VERSION}" fi @@ -181,10 +246,9 @@ if [ ${HELM_VERSION} != "none" ]; then get_helm "${HELM_VERSION}" if grep -q "BlobNotFound" "${tmp_helm_filename}"; then echo -e "\n(!) Failed to fetch the latest artifacts for helm ${HELM_VERSION}..." - repo_url=https://api.github.com/repos/helm/helm/releases - requested_version=$(get_previous_version "${repo_url}") - echo -e "\nAttempting to install ${requested_version}" - HELM_VERSION=${requested_version} + repo_url=$(get_github_api_repo_url "${helm_url}") + get_previous_version "${helm_url}" "${repo_url}" HELM_VERSION + echo -e "\nAttempting to install ${HELM_VERSION}" get_helm "${HELM_VERSION}" fi export GNUPGHOME="/tmp/helm/gnupg" diff --git a/test/kubectl-helm-minikube/install_only_helm_fallback.sh b/test/kubectl-helm-minikube/install_only_helm_fallback.sh index b8071a0b5..9f6a8e095 100644 --- a/test/kubectl-helm-minikube/install_only_helm_fallback.sh +++ b/test/kubectl-helm-minikube/install_only_helm_fallback.sh @@ -31,33 +31,118 @@ case $architecture in i?86) architecture="386";; *) echo "(!) Architecture $architecture unsupported"; exit 1 ;; esac -HELM_SHA256="${HELM_SHA256:-"automatic"}" -HELM_GPG_KEYS_URI="https://raw.githubusercontent.com/helm/helm/main/KEYS" -repo_url=https://api.github.com/repos/helm/helm/releases +helm_url="https://github.com/helm/helm" -# Function to fetch the latest version of the plugin -get_latest_version() { - curl -s "$repo_url/latest" | jq -r '.tag_name' +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" } -# Function to change the patch number in a semver version -change_patch_number() { - local version="$1" # Input version - local new_patch="$2" # New patch number - # Extract major, minor, and current patch numbers - local major=$(echo "$version" | cut -d. -f1) - local minor=$(echo "$version" | cut -d. -f2) - local current_patch=$(echo "$version" | cut -d. -f3) - # Construct the new version with the updated patch number - local new_version="$major.$minor.$new_patch" - echo "$new_version" +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e } -# Function to fetch the previous version of the plugin +# Function to fetch the version released prior to the latest version get_previous_version() { - # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name#v} + + output=$(curl -s "$repo_url"); + + message=$(echo "$output" | jq -r '.message') + + if [ $mode == "mode1" ]; then + message="API rate limit exceeded" + else + message="" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="v${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } get_helm() { @@ -68,53 +153,30 @@ get_helm() { sudo curl -sSL "https://github.com/helm/helm/releases/download/${HELM_VERSION}/${helm_filename}.asc" -o "${tmp_helm_filename}.asc" } -latest_version=$(get_latest_version) -NON_EXISTING_PATCH_VERSION="xyz" -HELM_VERSION="$(change_patch_number ${latest_version} ${NON_EXISTING_PATCH_VERSION})" -echo -e "\n๐Ÿ‘‰${HL} Trying to install HELM_VERSION = ${HELM_VERSION}${N}"; -sudo mkdir -p /tmp/helm -get_helm "${HELM_VERSION}" -if grep -q "BlobNotFound" "/tmp/helm/${helm_filename}"; then - echo -e "\n(!) Failed to fetch the latest artifacts for helm ${HELM_VERSION}..." - requested_version=$(get_previous_version) - echo -e "\nAttempting to install ${requested_version}" - HELM_VERSION=${requested_version} +install_helm() { + mode=$1 + HELM_VERSION="v3.14.xyz" + echo -e "\n๐Ÿ‘‰Trying to install HELM_VERSION = ${HELM_VERSION}"; + sudo mkdir -p /tmp/helm get_helm "${HELM_VERSION}" -fi -export GNUPGHOME="/tmp/helm/gnupg" -sudo mkdir -p "${GNUPGHOME}" -sudo chmod 700 ${GNUPGHOME} -sudo curl -sSL "${HELM_GPG_KEYS_URI}" -o /tmp/helm/KEYS -sudo echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" | sudo tee ${GNUPGHOME}/dirmngr.conf >/dev/null -sudo gpg -q --import "/tmp/helm/KEYS" -if ! sudo gpg --verify "${tmp_helm_filename}.asc" | sudo tee ${GNUPGHOME}/verify.log 2>&1; then - echo "Verification failed!" - sudo cat /tmp/helm/gnupg/verify.log - exit 1 -fi - -if [ "${HELM_SHA256}" = "automatic" ]; then - sudo curl -sSL "https://get.helm.sh/${helm_filename}.sha256" -o "${tmp_helm_filename}.sha256" - sudo curl -sSL "https://github.com/helm/helm/releases/download/${HELM_VERSION}/${helm_filename}.sha256.asc" -o "${tmp_helm_filename}.sha256.asc" - if ! sudo gpg --verify "${tmp_helm_filename}.sha256.asc" | sudo tee /tmp/helm/gnupg/verify.log 2>&1; then - echo "Verification failed!" - sudo cat /tmp/helm/gnupg/verify.log - exit 1 + if grep -q "BlobNotFound" "/tmp/helm/${helm_filename}"; then + echo -e "\n(!) Failed to fetch the latest artifacts for helm ${HELM_VERSION}..." + repo_url=$(get_github_api_repo_url "${helm_url}") + get_previous_version "${helm_url}" "${repo_url}" HELM_VERSION $mode + echo -e "\nAttempting to install ${HELM_VERSION}" + get_helm "${HELM_VERSION}" fi - HELM_SHA256="$(sudo cat "${tmp_helm_filename}.sha256")" -fi - -([ "${HELM_SHA256}" = "dev-mode" ] || (sudo echo "${HELM_SHA256} *${tmp_helm_filename}" | sha256sum -c -)) -sudo tar xf "${tmp_helm_filename}" -C /tmp/helm -sudo mv -f "/tmp/helm/linux-${architecture}/helm" /usr/local/bin/ -sudo chmod 0755 /usr/local/bin/helm -sudo rm -rf /tmp/helm -if ! type helm > /dev/null 2>&1; then - echo '(!) Helm installation failed!' - exit 1 -fi - -echo -e "\n๐Ÿ‘‰${HL} helm version as installed by test for fallback${N}:" +} + +echo -e "\n๐Ÿ‘‰${HL} helm version as installed by test for fallback${N}: (mode1: installation using find_prev_version_using_git_tags() fn)" +install_helm "mode1" + +set +e + check "helm version" helm version +set -e + +echo -e "\n๐Ÿ‘‰${HL} helm version as installed by test for fallback${N}: (mode2: installation using GitHub api)" +install_helm "mode2" set +e check "helm version" helm version From 32d2b5baf8071f00297fa0cb9fcfd79b9f238387 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 28 Mar 2024 23:00:49 +0530 Subject: [PATCH 028/247] [docker-outside-of-docker] - alternative fallback implementation (#924) --- .../devcontainer-feature.json | 2 +- src/docker-outside-of-docker/install.sh | 84 ++++++++- .../docker_build_compose_fallback.sh | 161 ++++++++++++++++-- 3 files changed, 224 insertions(+), 23 deletions(-) diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index 2a55fdbca..2506031ae 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.4.4", + "version": "1.4.5", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index 6b63469f9..16ad15f60 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -99,19 +99,84 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } -# Function to fetch the previous version of the plugin +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version get_previous_version() { - repo_url=$1 - # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" } +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} install_compose_switch_fallback() { + compose_switch_url=$1 + repo_url=$(get_github_api_repo_url "${compose_switch_url}") echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." - previous_version=$(get_previous_version "https://api.github.com/repos/docker/compose-switch/releases") - echo -e "\nAttempting to install ${previous_version}" - compose_switch_version=${previous_version#v} + get_previous_version "${compose_switch_url}" "${repo_url}" compose_switch_version + echo -e "\nAttempting to install v${compose_switch_version}" curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose } @@ -270,8 +335,9 @@ if [ "${DOCKER_DASH_COMPOSE_VERSION}" != "none" ]; then else echo "(*) Installing compose-switch as docker-compose..." compose_switch_version="latest" - find_version_from_git_tags compose_switch_version "https://github.com/docker/compose-switch" - curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback + compose_switch_url="https://github.com/docker/compose-switch" + find_version_from_git_tags compose_switch_version "${compose_switch_url}" + curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback "${compose_switch_url}" chmod +x /usr/local/bin/docker-compose # TODO: Verify checksum once available: https://github.com/docker/compose-switch/issues/11 fi diff --git a/test/docker-outside-of-docker/docker_build_compose_fallback.sh b/test/docker-outside-of-docker/docker_build_compose_fallback.sh index e1c0885e6..193d79d28 100644 --- a/test/docker-outside-of-docker/docker_build_compose_fallback.sh +++ b/test/docker-outside-of-docker/docker_build_compose_fallback.sh @@ -3,34 +3,169 @@ # Optional: Import test library source dev-container-features-test-lib +echo -e "\n๐Ÿ‘‰ Checking version of compose-switch installed as docker-compose as installed by feature"; check "installs compose-switch as docker-compose" bash -c "[[ -f /usr/local/bin/docker-compose ]]" +trap 'echo "Last executed command failed at line ${LINENO}"' ERR + # Fetch host/container arch. architecture="$(dpkg --print-architecture)" -repo_url="https://api.github.com/repos/docker/compose-switch/releases" +sudo mkdir -p /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" -# Function to fetch the previous version of the plugin + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version get_previous_version() { - sudo curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' # this would del the assets key and then get the second encountered tag_name's value from the filtered array of objects + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ $mode == 'mode1' ]]; then + message="API rate limit exceeded" + else + message="" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } install_compose_switch_fallback() { - echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch ${test_compose_switch_version}..." - previous_version=$(get_previous_version) - echo -e "\nAttempting to install ${previous_version}" - compose_switch_version=${previous_version} - sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose + compose_switch_url=$1 + mode=$2 + repo_url=$(get_github_api_repo_url "${compose_switch_url}") + echo -e "\n(!) Failed to fetch the latest artifacts for compose-switch v${compose_switch_version}..." + get_previous_version "${compose_switch_url}" "${repo_url}" compose_switch_version $mode + echo -e "\nAttempting to install v${compose_switch_version}" + sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose } install_compose-switch_as_docker-compose() { + mode=$1 echo "(*) Installing compose-switch as docker-compose..." - test_compose_switch_version="1.2.xyz" - echo -e "\nTesting with $test_compose_switch_version..." - sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/${test_compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback + compose_switch_version="1.0.6" + compose_switch_url="https://github.com/docker/compose-switch" + sudo curl -fsSL "https://github.com/docker/compose-switch/releases/download/v${compose_switch_version}/docker-compose-linux-${architecture}" -o /usr/local/bin/docker-compose || install_compose_switch_fallback "${compose_switch_url}" $mode sudo chmod +x /usr/local/bin/docker-compose } -install_compose-switch_as_docker-compose +echo -e "\n๐Ÿ‘‰ Trying to install compose-switch as docker-compose using mode 1 ( find_prev_version_from_git_tags method )"; +install_compose-switch_as_docker-compose "mode1" +check "installs compose-switch as docker-compose mode 1" bash -c "[[ -f /usr/local/bin/docker-compose ]]" -check "installs compose-switch as docker-compose" bash -c "[[ -f /usr/local/bin/docker-compose ]]" \ No newline at end of file +echo -e "\n๐Ÿ‘‰ Trying to install compose-switch as docker-compose using mode 2 ( GitHub Api )"; +install_compose-switch_as_docker-compose "mode2" +check "installs compose-switch as docker-compose mode 2" bash -c "[[ -f /usr/local/bin/docker-compose ]]" \ No newline at end of file From 55dbe138fc159c462a6c4554b4619db3ff3b04ab Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Tue, 2 Apr 2024 06:05:32 +0530 Subject: [PATCH 029/247] [PowerShell]- Fallback method with find_prev_vers_frm_git_tags() implemented (#926) * [PowerShell]- Fallback method with find_prev_vers_frm_git_tags() done * bumped the feature version --- src/powershell/devcontainer-feature.json | 2 +- src/powershell/install.sh | 81 ++++++++- .../install_powershell_fallback_test.sh | 164 ++++++++++++++---- 3 files changed, 201 insertions(+), 46 deletions(-) diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index dd7a96b90..82ef39a30 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.3.4", + "version": "1.3.5", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/powershell/install.sh b/src/powershell/install.sh index 4ba8bcf5c..533da6f37 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -106,17 +106,83 @@ install_using_apt() { apt-get install -yq powershell${version_suffix} || return 1 } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + # Function to fetch the version released prior to the latest version get_previous_version() { - repo_url=$1 - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + check_packages jq + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" } +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + + install_prev_pwsh() { + pwsh_url=$1 + repo_url=$(get_github_api_repo_url $pwsh_url) echo -e "\n(!) Failed to fetch the latest artifacts for powershell v${POWERSHELL_VERSION}..." - previous_version=$(get_previous_version "https://api.github.com/repos/PowerShell/PowerShell/releases") - echo -e "\nAttempting to install ${previous_version}" - POWERSHELL_VERSION="${previous_version#v}" + get_previous_version $pwsh_url $repo_url POWERSHELL_VERSION + echo -e "\nAttempting to install v${POWERSHELL_VERSION}" install_pwsh "${POWERSHELL_VERSION}" } @@ -138,10 +204,11 @@ install_using_github() { if [ "${architecture}" = "amd64" ]; then architecture="x64" fi - find_version_from_git_tags POWERSHELL_VERSION https://github.com/PowerShell/PowerShell + pwsh_url="https://github.com/PowerShell/PowerShell" + find_version_from_git_tags POWERSHELL_VERSION $pwsh_url install_pwsh "${POWERSHELL_VERSION}" if grep -q "Not Found" "${powershell_filename}"; then - install_prev_pwsh + install_prev_pwsh $pwsh_url fi # Ugly - but only way to get sha256 is to parse release HTML. Remove newlines and tags, then look for filename followed by 64 hex characters. diff --git a/test/powershell/install_powershell_fallback_test.sh b/test/powershell/install_powershell_fallback_test.sh index 852d7d103..39c8bc49e 100644 --- a/test/powershell/install_powershell_fallback_test.sh +++ b/test/powershell/install_powershell_fallback_test.sh @@ -15,16 +15,125 @@ check "Powershell version as installed by feature" bash -c "pwsh --version" . /etc/os-release architecture="$(dpkg --print-architecture)" +sudo mkdir -p /var/lib/apt/lists/ + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version get_previous_version() { - repo_url=$1 - curl -s "$repo_url" | jq -r 'del(.[].assets) | .[0].tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + message=$(echo "$output" | jq -r '.message') + + if [ $mode == "mode1" ]; then + message="API rate limit exceeded" + else + message="" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } install_prev_pwsh() { + local pwsh_url=$1 + local mode=$2 + local repo_url=$(get_github_api_repo_url $pwsh_url) echo -e "\n(!) Failed to fetch the latest artifacts for powershell v${POWERSHELL_VERSION}..." - previous_version=$(get_previous_version "https://api.github.com/repos/PowerShell/PowerShell/releases") - echo -e "\nAttempting to install ${previous_version}" - POWERSHELL_VERSION="${previous_version#v}" + get_previous_version $pwsh_url $repo_url POWERSHELL_VERSION $mode + echo -e "\nAttempting to install v${POWERSHELL_VERSION}" install_pwsh "${POWERSHELL_VERSION}" } @@ -37,44 +146,18 @@ install_pwsh() { sudo curl -sSL -o "${powershell_filename}" "https://github.com/PowerShell/PowerShell/releases/download/v${POWERSHELL_VERSION}/${powershell_filename}" } -apt_get_update() -{ - if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then - echo "Running apt-get update..." - sudo apt-get update -y - fi -} - -check_packages() { - if ! dpkg -s "$@" > /dev/null 2>&1; then - sudo chmod +x /var/lib/apt/lists/ - sudo mkdir -p /var/lib/apt/lists/partial - sudo chmod +rx /var/lib/dpkg/lock-frontend - apt_get_update - sudo apt-get -y install --no-install-recommends "$@" - fi -} - install_using_github() { - # Fall back on direct download if no apt package exists in microsoft pool - check_packages curl ca-certificates gnupg2 dirmngr libc6 libgcc1 libgssapi-krb5-2 libstdc++6 libunwind8 libuuid1 zlib1g libicu[0-9][0-9] - if ! type git > /dev/null 2>&1; then - check_packages git - fi + mode=$1 if [ "${architecture}" = "amd64" ]; then architecture="x64" fi - - echo -e "\nTrying to install a non-existing version for Powershell..." - - POWERSHELL_VERSION="1.2.XYZ" + pwsh_url="https://github.com/PowerShell/PowerShell" + POWERSHELL_VERSION="7.4.xyz" install_pwsh "${POWERSHELL_VERSION}" - if grep -q "Not Found" "${powershell_filename}"; then - install_prev_pwsh + install_prev_pwsh $pwsh_url $mode fi - echo -e "\n" $POWERSHELL_VERSION "=powershell_version\n"; # Ugly - but only way to get sha256 is to parse release HTML. Remove newlines and tags, then look for filename followed by 64 hex characters. sudo curl -sSL -o "release.html" "https://github.com/PowerShell/PowerShell/releases/tag/v${POWERSHELL_VERSION}" powershell_archive_sha256="$(cat release.html | tr '\n' ' ' | sed 's|<[^>]*>||g' | grep -oP "${powershell_filename}\s+\K[0-9a-fA-F]{64}" || echo '')" @@ -86,12 +169,17 @@ install_using_github() { fi sudo tar xf "${powershell_filename}" -C "${powershell_target_path}" sudo ln -s "${powershell_target_path}/pwsh" /usr/local/bin/pwsh - sudo rm -rf /tmp/pwsh + sudo rm -rf /tmp/pwsh /usr/local/bin/pwsh + } -install_using_github +echo -e "\nInstalling Powershell with find_prev_version_from_git_tags() fn ๐Ÿ‘ˆ๐Ÿป" +install_using_github "mode1" +check "Powershell version as installed by test (find_prev_version_from_git_tags() fn)" bash -c "pwsh --version" -check "Powershell version as installed by test" bash -c "pwsh --version" +echo -e "\nInstalling Powershell with GitHub Api ๐Ÿ‘ˆ๐Ÿป" +install_using_github "mode2" +check "Powershell version as installed by test (GitHub Api)" bash -c "pwsh --version" # Report result reportResults From a7c8c9a83b3ce0f7f0c546537d814009f9e48590 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 3 Apr 2024 03:06:19 +0530 Subject: [PATCH 030/247] [Php] - fallback to previous version - code fix (#908) * [php]- php fallback to prev. version - fix * update patch version for php feature * few changes.. * small change * test completes successfully pointing at the new fallbacked php version * changes acc. to review comments.. --- src/php/devcontainer-feature.json | 2 +- src/php/install.sh | 69 +++++++++- test/php/scenarios.json | 8 ++ test/php/test_php_fallback.sh | 201 ++++++++++++++++++++++++++++++ 4 files changed, 274 insertions(+), 6 deletions(-) create mode 100644 test/php/test_php_fallback.sh diff --git a/src/php/devcontainer-feature.json b/src/php/devcontainer-feature.json index 77fcf9812..a4edcca95 100644 --- a/src/php/devcontainer-feature.json +++ b/src/php/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "php", - "version": "1.1.2", + "version": "1.1.3", "name": "PHP", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/php", "options": { diff --git a/src/php/install.sh b/src/php/install.sh index 48140e1b3..357395e88 100755 --- a/src/php/install.sh +++ b/src/php/install.sh @@ -121,6 +121,46 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + # Install PHP Composer addcomposer() { "${PHP_SRC}" -r "copy('https://getcomposer.org/installer', 'composer-setup.php');" @@ -130,8 +170,7 @@ addcomposer() { "${PHP_SRC}" -r "unlink('composer-setup.php');" } -install_php() { - PHP_VERSION="$1" +init_php_install() { PHP_INSTALL_DIR="${PHP_DIR}/${PHP_VERSION}" if [ -d "${PHP_INSTALL_DIR}" ]; then echo "(!) PHP version ${PHP_VERSION} already exists." @@ -142,7 +181,6 @@ install_php() { groupadd -r php fi usermod -a -G php "${USERNAME}" - PHP_URL="https://www.php.net/distributions/php-${PHP_VERSION}.tar.gz" PHP_INI_DIR="${PHP_INSTALL_DIR}/ini" @@ -155,7 +193,26 @@ install_php() { PHP_SRC_DIR="/usr/src/php" mkdir -p $PHP_SRC_DIR cd $PHP_SRC_DIR - wget -O php.tar.xz "$PHP_URL" +} + +install_previous_version() { + PHP_VERSION=$1 + if [[ "$ORIGINAL_PHP_VERSION" == "latest" ]]; then + find_prev_version_from_git_tags PHP_VERSION https://github.com/php/php-src "tags/php-" + echo -e "\nAttempting to install previous version v${PHP_VERSION}" + init_php_install + wget -O php.tar.xz "$PHP_URL" + else + echo -e "\nFailed to install v$PHP_VERSION" + fi +} + +install_php() { + PHP_VERSION="$1" + + init_php_install + + wget -O php.tar.xz "$PHP_URL" || install_previous_version "$PHP_VERSION" tar -xf $PHP_SRC_DIR/php.tar.xz -C "$PHP_SRC_DIR" --strip-components=1 cd $PHP_SRC_DIR; @@ -195,7 +252,7 @@ install_php() { if [ "${PHP_VERSION}" != "none" ]; then # Persistent / runtime dependencies - RUNTIME_DEPS="wget ca-certificates git build-essential xz-utils" + RUNTIME_DEPS="wget ca-certificates git build-essential xz-utils curl" # PHP dependencies PHP_DEPS="libssl-dev libcurl4-openssl-dev libedit-dev libsqlite3-dev libxml2-dev zlib1g-dev libsodium-dev libonig-dev" @@ -214,6 +271,8 @@ if [ "${PHP_VERSION}" != "none" ]; then # Install dependencies check_packages $RUNTIME_DEPS $PHP_DEPS $PHPIZE_DEPS + # storing value of PHP_VERSION before it changes + ORIGINAL_PHP_VERSION=$PHP_VERSION find_version_from_git_tags PHP_VERSION https://github.com/php/php-src "tags/php-" install_php "${PHP_VERSION}" diff --git a/test/php/scenarios.json b/test/php/scenarios.json index 1e4df063d..e53bb67ce 100644 --- a/test/php/scenarios.json +++ b/test/php/scenarios.json @@ -32,5 +32,13 @@ "installComposer": true } } + }, + "test_php_fallback": { + "image": "ubuntu:focal", + "features": { + "php": { + "version": "latest" + } + } } } diff --git a/test/php/test_php_fallback.sh b/test/php/test_php_fallback.sh new file mode 100644 index 000000000..84ed83d77 --- /dev/null +++ b/test/php/test_php_fallback.sh @@ -0,0 +1,201 @@ +#!/bin/bash + +echo -e "\nInstalled PHP Version by Feature: ๐Ÿ‘‡ "; php -v; + +USERNAME="root" +PHP_DIR="/usr/local/php" + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + echo "${!variable_name}" + echo "$(echo "${requested_version}" | grep -o "." | wc -l)" + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + echo "${!variable_name}" + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +init_php_install() { + PHP_INSTALL_DIR="${PHP_DIR}/${PHP_VERSION}" + if [ -d "${PHP_INSTALL_DIR}" ]; then + echo "(!) PHP version ${PHP_VERSION} already exists." + exit 1 + fi + + if ! cat /etc/group | grep -e "^php:" > /dev/null 2>&1; then + groupadd -r php + fi + usermod -a -G php "${USERNAME}" + + PHP_URL="https://www.php.net/distributions/php-${PHP_VERSION}.tar.gz" + + PHP_INI_DIR="${PHP_INSTALL_DIR}/ini" + CONF_DIR="${PHP_INI_DIR}/conf.d" + mkdir -p "${CONF_DIR}"; + + PHP_EXT_DIR="${PHP_INSTALL_DIR}/extensions" + mkdir -p "${PHP_EXT_DIR}" + + PHP_SRC_DIR="/usr/src/php" + mkdir -p $PHP_SRC_DIR + cd $PHP_SRC_DIR +} + +install_previous_version() { + echo -e "\nInstalling Previous Version..." + find_prev_version_from_git_tags PHP_VERSION https://github.com/php/php-src "tags/php-" + echo -e "\nNow installing this version as a fallback previous version: ${PHP_VERSION} ๐Ÿคž๐Ÿป" + init_php_install + wget -O php.tar.xz "$PHP_URL" +} + +install_php() { + # trying to install with a possible new tag not having a released source binary yet + PHP_VERSION="8.3.xyz" + + init_php_install + + wget -O php.tar.xz "$PHP_URL" || install_previous_version + + tar -xf $PHP_SRC_DIR/php.tar.xz -C "$PHP_SRC_DIR" --strip-components=1 + cd $PHP_SRC_DIR; + + # PHP 7.4+, the pecl/pear installers are officially deprecated and are removed in PHP 8+ + # Thus, requiring an explicit "--with-pear" + IFS="." + read -a versions <<< "${PHP_VERSION}" + PHP_MAJOR_VERSION=${versions[0]} + PHP_MINOR_VERSION=${versions[1]} + + VERSION_CONFIG="" + if (( $(($PHP_MAJOR_VERSION)) >= 8 )) || (( $(($PHP_MAJOR_VERSION)) == 7 && $(($PHP_MINOR_VERSION)) >= 4 )); then + VERSION_CONFIG="--with-pear" + fi + + ./configure --prefix="${PHP_INSTALL_DIR}" --with-config-file-path="$PHP_INI_DIR" --with-config-file-scan-dir="$CONF_DIR" --enable-option-checking=fatal --with-curl --with-libedit --enable-mbstring --with-openssl --with-zlib --with-password-argon2 --with-sodium=shared "$VERSION_CONFIG" EXTENSION_DIR="$PHP_EXT_DIR"; + + make -j "$(nproc)" + find -type f -name '*.a' -delete + make install + find "${PHP_INSTALL_DIR}" -type f -executable -exec strip --strip-all '{}' + || true + make clean + + cp -v $PHP_SRC_DIR/php.ini-* "$PHP_INI_DIR/"; + cp "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" + + # Install xdebug + "${PHP_INSTALL_DIR}/bin/pecl" install xdebug + XDEBUG_INI="${CONF_DIR}/xdebug.ini" + + echo "zend_extension=${PHP_EXT_DIR}/xdebug.so" > "${XDEBUG_INI}" + echo "xdebug.mode = debug" >> "${XDEBUG_INI}" + echo "xdebug.start_with_request = yes" >> "${XDEBUG_INI}" + echo "xdebug.client_port = 9003" >> "${XDEBUG_INI}" +} + +apt-get purge php.* +PHP_DIR="/usr/local/php" +PHP_INSTALL_DIR="${PHP_DIR}/${PHP_VERSION}" +PHP_SRC_DIR="/usr/src/php" + +install_php +PHP_SRC="${PHP_INSTALL_DIR}/bin/php" + +updaterc() { + echo "Updating /etc/bash.bashrc and /etc/zsh/zshrc..." + if [[ "$(cat /etc/bash.bashrc)" != *"$1"* ]]; then + echo -e "$1" >> /etc/bash.bashrc + fi + if [ -f "/etc/zsh/zshrc" ] && [[ "$(cat /etc/zsh/zshrc)" != *"$1"* ]]; then + echo -e "$1" >> /etc/zsh/zshrc + fi +} + +if [ "${PHP_VERSION}" != "none" ]; then + CURRENT_DIR="${PHP_DIR}/current" + if [[ ! -d "${CURRENT_DIR}" ]]; then + ln -s -r "${PHP_INSTALL_DIR}" ${CURRENT_DIR} + fi + + if [[ $(ls -l ${CURRENT_DIR}) != *"-> ${PHP_INSTALL_DIR}"* ]] ; then + rm "${CURRENT_DIR}" + ln -s -r "${PHP_INSTALL_DIR}" "${CURRENT_DIR}" + fi + + rm -rf "${PHP_SRC_DIR}" + updaterc "if [[ \"\${PATH}\" != *\"${CURRENT_DIR}\"* ]]; then export PATH=\"${CURRENT_DIR}/bin:\${PATH}\"; fi" + + chown -R "${USERNAME}:php" "${PHP_DIR}" + chmod -R g+r+w "${PHP_DIR}" + find "${PHP_DIR}" -type d -print0 | xargs -n 1 -0 chmod g+s +fi + +echo -e "\nInstalled PHP Version by Test: ๐Ÿ‘‡ "; php -v; + From 9ccc19e1378ba3b569784236bbb32499dcea138e Mon Sep 17 00:00:00 2001 From: WarrenS Date: Tue, 2 Apr 2024 17:42:58 -0400 Subject: [PATCH 031/247] Added versions 1.71-1.76 to Rust feature version (#929) * Update devcontainer-feature.json * Updated version --- src/rust/devcontainer-feature.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/rust/devcontainer-feature.json b/src/rust/devcontainer-feature.json index 7ee455f3f..c4a7dd5b9 100644 --- a/src/rust/devcontainer-feature.json +++ b/src/rust/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "rust", - "version": "1.1.1", + "version": "1.1.2", "name": "Rust", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/rust", "description": "Installs Rust, common Rust utilities, and their required dependencies", @@ -10,6 +10,12 @@ "proposals": [ "latest", "none", + "1.76", + "1.75", + "1.74", + "1.73", + "1.72", + "1.71", "1.70", "1.69", "1.68", From 203dc3f5bde1a8ca25525234757ac54e9b8da64c Mon Sep 17 00:00:00 2001 From: Prathamesh Zarkar <159782310+prathameshzarkar9@users.noreply.github.com> Date: Fri, 5 Apr 2024 04:23:20 +0530 Subject: [PATCH 032/247] oryx dotnet 8.0.1 cleanup (#927) --- src/oryx/devcontainer-feature.json | 2 +- src/oryx/install.sh | 1 + test/oryx/test_python_project.sh | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/oryx/devcontainer-feature.json b/src/oryx/devcontainer-feature.json index a075096bf..41bd1f759 100644 --- a/src/oryx/devcontainer-feature.json +++ b/src/oryx/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "oryx", - "version": "1.3.1", + "version": "1.3.2", "name": "Oryx", "description": "Installs the oryx CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/oryx", diff --git a/src/oryx/install.sh b/src/oryx/install.sh index 407406141..6c193f863 100755 --- a/src/oryx/install.sh +++ b/src/oryx/install.sh @@ -241,6 +241,7 @@ if [[ "${PINNED_SDK_VERSION}" != "" ]]; then MAJOR_MINOR_PATCH1_VERSION=${PINNED_SDK_VERSION%??} rm -rf /usr/share/dotnet/shared/Microsoft.NETCore.App/$MAJOR_MINOR_PATCH1_VERSION rm -rf /usr/share/dotnet/shared/Microsoft.AspNetCore.App/$MAJOR_MINOR_PATCH1_VERSION + rm -rf /usr/share/dotnet/templates/$MAJOR_MINOR_PATCH1_VERSION fi diff --git a/test/oryx/test_python_project.sh b/test/oryx/test_python_project.sh index a0d5eef08..610a9687e 100644 --- a/test/oryx/test_python_project.sh +++ b/test/oryx/test_python_project.sh @@ -28,5 +28,7 @@ check "oryx-build-python" oryx build --property python_version="${pythonVersion} check "oryx-build-python-installed" python3 -m pip list | grep mpmath check "oryx-build-python-result" python3 ./src/solve.py +check "templates/8.0.1-does-not-exist" test ! -d "/usr/share/dotnet/templates/8.0.1" + # Report result reportResults From 760f2bf10b30340e55a2cdd9862a994262f13d81 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste Louazel Date: Thu, 11 Apr 2024 18:40:05 +0200 Subject: [PATCH 033/247] Set `DEBIAN_FRONTEND=noninteractive` for nvidia-cuda feature (#933) * Set DEBIAN_FRONTEND to noninteractive Signed-off-by: Jean-Baptiste Louazel * Bump nvidia-cuda to 1.1.1 Signed-off-by: Jean-Baptiste Louazel * Install `liburcu6` Signed-off-by: Jean-Baptiste Louazel * Revert "Install `liburcu6`" This reverts commit b7b2931f8bed57826f2019ea884cbe2440e6c9e8. --------- Signed-off-by: Jean-Baptiste Louazel --- src/nvidia-cuda/devcontainer-feature.json | 2 +- src/nvidia-cuda/install.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nvidia-cuda/devcontainer-feature.json b/src/nvidia-cuda/devcontainer-feature.json index 78ad10cd9..bb63ae1c7 100644 --- a/src/nvidia-cuda/devcontainer-feature.json +++ b/src/nvidia-cuda/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "nvidia-cuda", - "version": "1.1.0", + "version": "1.1.1", "name": "NVIDIA CUDA", "description": "Installs shared libraries for NVIDIA CUDA.", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/nvidia-cuda", diff --git a/src/nvidia-cuda/install.sh b/src/nvidia-cuda/install.sh index cb66d3955..d8658964e 100644 --- a/src/nvidia-cuda/install.sh +++ b/src/nvidia-cuda/install.sh @@ -33,6 +33,8 @@ check_packages() { fi } +export DEBIAN_FRONTEND=noninteractive + check_packages wget ca-certificates # Add NVIDIA's package repository to apt so that we can download packages From e7dd9fafd9aeede11d0a59a0ace819e3b774de2a Mon Sep 17 00:00:00 2001 From: Prathamesh Zarkar <159782310+prathameshzarkar9@users.noreply.github.com> Date: Fri, 12 Apr 2024 23:33:24 +0530 Subject: [PATCH 034/247] cp command to follow symlink for systemctl (#937) --- src/common-utils/devcontainer-feature.json | 2 +- src/common-utils/main.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common-utils/devcontainer-feature.json b/src/common-utils/devcontainer-feature.json index a058864af..308e256ef 100644 --- a/src/common-utils/devcontainer-feature.json +++ b/src/common-utils/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "common-utils", - "version": "2.4.2", + "version": "2.4.3", "name": "Common Utilities", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/common-utils", "description": "Installs a set of common command line utilities, Oh My Zsh!, and sets up a non-root user.", diff --git a/src/common-utils/main.sh b/src/common-utils/main.sh index 8b74830cc..5d7592cfc 100644 --- a/src/common-utils/main.sh +++ b/src/common-utils/main.sh @@ -564,7 +564,7 @@ chmod +rx /usr/local/bin/code # systemctl shim for Debian/Ubuntu - tells people to use 'service' if systemd is not running if [ "${ADJUSTED_ID}" = "debian" ]; then - cp -f "${FEATURE_DIR}/bin/systemctl" /usr/local/bin/systemctl + cp -fL "${FEATURE_DIR}/bin/systemctl" /usr/local/bin/systemctl chmod +rx /usr/local/bin/systemctl fi From b98f5a164be78317af27118e9491d38c17eb16a4 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 17 Apr 2024 04:28:57 +0530 Subject: [PATCH 035/247] [az-cli] - To separate the two methods of installation - using apt, using python (#922) * [az-cli] - To separate the two methods of installation - apt & python * bump patch version * changes as requested by review comments * no need to keep * added test script missing * changes for review comment --- src/azure-cli/devcontainer-feature.json | 2 +- src/azure-cli/install.sh | 10 +++++---- ..._using_python_with_python_3_11_bullseye.sh | 21 +++++++++++++++++++ .../install_with_python_3_12_bookworm.sh | 12 +++++------ test/azure-cli/scenarios.json | 10 +++++++++ 5 files changed, 44 insertions(+), 11 deletions(-) create mode 100644 test/azure-cli/install_using_python_with_python_3_11_bullseye.sh diff --git a/src/azure-cli/devcontainer-feature.json b/src/azure-cli/devcontainer-feature.json index f25e5f9a2..6b26ef6fc 100644 --- a/src/azure-cli/devcontainer-feature.json +++ b/src/azure-cli/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "azure-cli", - "version": "1.2.3", + "version": "1.2.4", "name": "Azure CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/azure-cli", "description": "Installs the Azure CLI along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/azure-cli/install.sh b/src/azure-cli/install.sh index 7f4c4ff1c..a1b254779 100755 --- a/src/azure-cli/install.sh +++ b/src/azure-cli/install.sh @@ -15,7 +15,7 @@ rm -rf /var/lib/apt/lists/* AZ_VERSION=${VERSION:-"latest"} AZ_EXTENSIONS=${EXTENSIONS} AZ_INSTALLBICEP=${INSTALLBICEP:-false} -INSTALL_USING_PYTHON=${INSTALL_USING_PYTHON:-false} +INSTALL_USING_PYTHON=${INSTALLUSINGPYTHON:-false} MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" AZCLI_ARCHIVE_ARCHITECTURES="amd64 arm64" AZCLI_ARCHIVE_VERSION_CODENAMES="stretch bookworm buster bullseye bionic focal jammy" @@ -188,13 +188,15 @@ echo "(*) Installing Azure CLI..." . /etc/os-release architecture="$(dpkg --print-architecture)" CACHED_AZURE_VERSION="${AZ_VERSION}" # In case we need to fallback to pip and the apt path has modified the AZ_VERSION variable. -if [[ "${AZCLI_ARCHIVE_ARCHITECTURES}" = *"${architecture}"* ]] && [[ "${AZCLI_ARCHIVE_VERSION_CODENAMES}" = *"${VERSION_CODENAME}"* ]]; then - install_using_apt || use_pip="true" +if [ "${INSTALL_USING_PYTHON}" != "true" ]; then + if [[ "${AZCLI_ARCHIVE_ARCHITECTURES}" = *"${architecture}"* ]] && [[ "${AZCLI_ARCHIVE_VERSION_CODENAMES}" = *"${VERSION_CODENAME}"* ]]; then + install_using_apt || use_pip="true" + fi else use_pip="true" fi -if [ "${use_pip}" = "true" ]; then +if [ "${use_pip}" = "true" ]; then AZ_VERSION=${CACHED_AZURE_VERSION} install_using_pip_strategy diff --git a/test/azure-cli/install_using_python_with_python_3_11_bullseye.sh b/test/azure-cli/install_using_python_with_python_3_11_bullseye.sh new file mode 100644 index 000000000..b9957843e --- /dev/null +++ b/test/azure-cli/install_using_python_with_python_3_11_bullseye.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode +check "version" az --version + +echo -e "\n\n๐Ÿ”„ Testing 'O.S'" +if cat /etc/os-release | grep -q 'PRETTY_NAME="Debian GNU/Linux 11 (bullseye)"'; then + echo -e "\n\nโœ… Passed 'O.S is Linux 11 (bullseye)'!" +else + echo -e "\n\nโŒ Failed 'O.S is other than Linux 11 (bullseye)'!" +fi + + +# Report result +reportResults \ No newline at end of file diff --git a/test/azure-cli/install_with_python_3_12_bookworm.sh b/test/azure-cli/install_with_python_3_12_bookworm.sh index 074876148..2c8e1fd72 100644 --- a/test/azure-cli/install_with_python_3_12_bookworm.sh +++ b/test/azure-cli/install_with_python_3_12_bookworm.sh @@ -5,17 +5,17 @@ set -e # Import test library for `check` command source dev-container-features-test-lib -# Check to make sure the user is vscode -check "user is vscode" whoami | grep vscode -check "version" az --version -echo -e "\n\n๐Ÿ”„ Testing 'O.S'" +echo -e "\n๐Ÿ”„ Testing 'O.S'" if cat /etc/os-release | grep -q 'PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"'; then - echo -e "\n\nโœ… Passed 'O.S is Linux 12 (bookworm)'!" + echo -e "\nโœ… Passed 'O.S is Linux 12 (bookworm)'!\n" else - echo -e "\n\nโŒ Failed 'O.S is other than Linux 12 (bookworm)'!" + echo -e "\nโŒ Failed 'O.S is other than Linux 12 (bookworm)'!\n" fi +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode +check "version" az --version # Report result reportResults \ No newline at end of file diff --git a/test/azure-cli/scenarios.json b/test/azure-cli/scenarios.json index 041f50731..3ec910399 100644 --- a/test/azure-cli/scenarios.json +++ b/test/azure-cli/scenarios.json @@ -47,5 +47,15 @@ "version": "latest" } } + }, + "install_using_python_with_python_3_11_bullseye": { + "image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye", + "user": "vscode", + "features": { + "azure-cli": { + "version": "latest", + "installUsingPython": "true" + } + } } } \ No newline at end of file From 6f4e59866169405c7b7a8ff65e3f2ac3ced6a26e Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 17 Apr 2024 04:35:23 +0530 Subject: [PATCH 036/247] [Ruby]- rvm - fallback code fix (#931) * [Ruby] - Install using fallback - draft * [Ruby] - Rvm - fallback logic implementation * misc changes * changes for review comments.. --- src/ruby/devcontainer-feature.json | 2 +- src/ruby/install.sh | 129 ++++++++++++-- test/ruby/ruby_fallback_test.sh | 277 +++++++++++++++++++++++++++++ test/ruby/scenarios.json | 8 + 4 files changed, 396 insertions(+), 20 deletions(-) create mode 100644 test/ruby/ruby_fallback_test.sh diff --git a/src/ruby/devcontainer-feature.json b/src/ruby/devcontainer-feature.json index 73bcbcced..3722cab06 100644 --- a/src/ruby/devcontainer-feature.json +++ b/src/ruby/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "ruby", - "version": "1.2.0", + "version": "1.2.1", "name": "Ruby (via rvm)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/ruby", "description": "Installs Ruby, rvm, rbenv, common Ruby utilities, and needed dependencies.", diff --git a/src/ruby/install.sh b/src/ruby/install.sh index 7e4514bba..8f95829da 100755 --- a/src/ruby/install.sh +++ b/src/ruby/install.sh @@ -140,6 +140,47 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + apt_get_update() { if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then @@ -173,25 +214,46 @@ if ! type git > /dev/null 2>&1; then check_packages git fi +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + #install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" "_" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name' | tr '_' '.') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + # Figure out correct version of a three part version number is not passed -find_version_from_git_tags RUBY_VERSION "https://github.com/ruby/ruby" "tags/v" "_" +RUBY_URL="https://github.com/ruby/ruby" +ORIGINAL_RUBY_VERSION=$RUBY_VERSION +find_version_from_git_tags RUBY_VERSION $RUBY_URL "tags/v" "_" -# Just install Ruby if RVM already installed -if rvm --version > /dev/null; then - echo "Ruby Version Manager already exists." - if [[ "$(ruby -v)" = *"${RUBY_VERSION}"* ]]; then - echo "(!) Ruby is already installed with version ${RUBY_VERSION}. Skipping..." - elif [ "${RUBY_VERSION}" != "none" ]; then - echo "Installing specified Ruby version." - su ${USERNAME} -c "rvm install ruby ${RUBY_VERSION}" - fi - SKIP_GEM_INSTALL="false" - SKIP_RBENV_RBUILD="true" -else - # Install RVM - receive_gpg_keys RVM_GPG_KEYS - # Determine appropriate settings for rvm installer +set_rvm_install_args() { + RUBY_VERSION=$1 if [ "${RUBY_VERSION}" = "none" ]; then RVM_INSTALL_ARGS="" elif [[ "$(ruby -v)" = *"${RUBY_VERSION}"* ]]; then @@ -210,19 +272,48 @@ else DEFAULT_GEMS="" fi fi +} + +install_previous_version() { + if [[ $ORIGINAL_RUBY_VERSION == "latest" ]]; then + repo_url=$(get_github_api_repo_url "$RUBY_URL") + get_previous_version "${RUBY_URL}" "${repo_url}" RUBY_VERSION + set_rvm_install_args $RUBY_VERSION + curl -sSL https://get.rvm.io | bash -s stable --ignore-dotfiles ${RVM_INSTALL_ARGS} --with-default-gems="${DEFAULT_GEMS}" 2>&1 + else + echo "Failed to install Ruby version $ORIGINAL_RUBY_VERSION. Exiting..." + fi +} + +# Just install Ruby if RVM already installed +if rvm --version > /dev/null; then + echo "Ruby Version Manager already exists." + if [[ "$(ruby -v)" = *"${RUBY_VERSION}"* ]]; then + echo "(!) Ruby is already installed with version ${RUBY_VERSION}. Skipping..." + elif [ "${RUBY_VERSION}" != "none" ]; then + echo "Installing specified Ruby version." + su ${USERNAME} -c "rvm install ruby ${RUBY_VERSION}" + fi + SKIP_GEM_INSTALL="false" + SKIP_RBENV_RBUILD="true" +else + # Install RVM + receive_gpg_keys RVM_GPG_KEYS + # Determine appropriate settings for rvm installer + set_rvm_install_args $RUBY_VERSION # Create rvm group as a system group to reduce the odds of conflict with local user UIDs if ! cat /etc/group | grep -e "^rvm:" > /dev/null 2>&1; then groupadd -r rvm fi # Install rvm - curl -sSL https://get.rvm.io | bash -s stable --ignore-dotfiles ${RVM_INSTALL_ARGS} --with-default-gems="${DEFAULT_GEMS}" 2>&1 + curl -sSL https://get.rvm.io | bash -s stable --ignore-dotfiles ${RVM_INSTALL_ARGS} --with-default-gems="${DEFAULT_GEMS}" 2>&1 || install_previous_version usermod -aG rvm ${USERNAME} source /usr/local/rvm/scripts/rvm rvm fix-permissions system rm -rf ${GNUPGHOME} fi -if [ "${INSTALL_RUBY_TOOLS}" = "true" ]; then +if [ "${INSTALL_RUBY_TOOLS}" = "true" ]; then # Non-root user may not have "gem" in path when script is run and no ruby version # is installed by rvm, so handle this by using root's default gem in this case ROOT_GEM="$(which gem || echo "")" @@ -239,7 +330,7 @@ if [ ! -z "${ADDITIONAL_VERSIONS}" ]; then read -a additional_versions <<< "$ADDITIONAL_VERSIONS" for version in "${additional_versions[@]}"; do # Figure out correct version of a three part version number is not passed - find_version_from_git_tags version "https://github.com/ruby/ruby" "tags/v" "_" + find_version_from_git_tags version $RUBY_URL "tags/v" "_" source /usr/local/rvm/scripts/rvm rvm install ruby ${version} done diff --git a/test/ruby/ruby_fallback_test.sh b/test/ruby/ruby_fallback_test.sh new file mode 100644 index 000000000..fe1a9c77b --- /dev/null +++ b/test/ruby/ruby_fallback_test.sh @@ -0,0 +1,277 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +USERNAME="automatic" +echo -e "\nRVM version installed previously by ruby feature ..." +check "rvm" rvm --version +check "ruby" ruby -v + +trap 'echo "Last executed command failed at line ${LINENO}"' ERR + +RVM_GPG_KEYS="409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB" +GPG_KEY_SERVERS="keyserver hkp://keyserver.ubuntu.com +keyserver hkp://keyserver.ubuntu.com:80 +keyserver hkps://keys.openpgp.org +keyserver hkp://keyserver.pgp.com" + +# Clean up +rm -rf /var/lib/apt/lists/* + +# Determine the appropriate non-root user +if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then + USERNAME="" + POSSIBLE_USERS=("vscode" "node" "codespace" "$(awk -v val=1000 -F ":" '$3==val{print $1}' /etc/passwd)") + for CURRENT_USER in "${POSSIBLE_USERS[@]}"; do + if id -u ${CURRENT_USER} > /dev/null 2>&1; then + USERNAME=${CURRENT_USER} + break + fi + done + if [ "${USERNAME}" = "" ]; then + USERNAME=root + fi +elif [ "${USERNAME}" = "none" ] || ! id -u ${USERNAME} > /dev/null 2>&1; then + USERNAME=root +fi + +# Ensure apt is in non-interactive to avoid prompts +export DEBIAN_FRONTEND=noninteractive + +architecture="$(uname -m)" +if [ "${architecture}" != "amd64" ] && [ "${architecture}" != "x86_64" ] && [ "${architecture}" != "arm64" ] && [ "${architecture}" != "aarch64" ]; then + echo "(!) Architecture $architecture unsupported" + exit 1 +fi + +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + +# Import the specified key in a variable name passed in as +receive_gpg_keys() { + local keys=${!1} + local keyring_args="" + if [ ! -z "$2" ]; then + keyring_args="--no-default-keyring --keyring \"$2\"" + fi + + # Use a temporary location for gpg keys to avoid polluting image + export GNUPGHOME="/tmp/tmp-gnupg" + mkdir -p ${GNUPGHOME} + chmod 700 ${GNUPGHOME} + echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" | tee ${GNUPGHOME}/dirmngr.conf > /dev/null + # GPG key download sometimes fails for some reason and retrying fixes it. + local retry_count=0 + local gpg_ok="false" + set +e + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; + do + echo "(*) Downloading GPG key..." + ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" + if [ "${gpg_ok}" != "true" ]; then + echo "(*) Failed getting key, retring in 10s..." + (( retry_count++ )) + sleep 10s + fi + done + set -e + if [ "${gpg_ok}" = "false" ]; then + echo "(!) Failed to get gpg key." + exit 1 + fi +} + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + + #install jq + check_packages jq + + message=$(echo "$output" | jq -r '.message') + + if [[ $mode == "mode1" ]]; then + message="API rate limit exceeded" + else + message="" + fi + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" "_" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name' | tr '_' '.') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + + +# Figure out correct version of a three part version number is not passed +ruby_url="https://github.com/ruby/ruby" + +RUBY_VERSION="3.1.xyz" + +set_rvm_install_args() { + RUBY_VERSION=$1 + if [ "${RUBY_VERSION}" = "none" ]; then + RVM_INSTALL_ARGS="" + elif [[ "$(ruby -v)" = *"${RUBY_VERSION}"* ]]; then + echo "(!) Ruby is already installed with version ${RUBY_VERSION}. Skipping..." + RVM_INSTALL_ARGS="" + else + if [ "${RUBY_VERSION}" = "latest" ] || [ "${RUBY_VERSION}" = "current" ] || [ "${RUBY_VERSION}" = "lts" ]; then + RVM_INSTALL_ARGS="--ruby" + RUBY_VERSION="" + else + RVM_INSTALL_ARGS="--ruby=${RUBY_VERSION}" + fi + if [ "${INSTALL_RUBY_TOOLS}" = "true" ]; then + SKIP_GEM_INSTALL="true" + else + DEFAULT_GEMS="" + fi + fi +} + +install_previous_version() { + mode=$1 + repo_url=$(get_github_api_repo_url "$ruby_url") + get_previous_version "${ruby_url}" "${repo_url}" RUBY_VERSION $mode + set_rvm_install_args $RUBY_VERSION + curl -sSL https://get.rvm.io | bash -s stable --ignore-dotfiles ${RVM_INSTALL_ARGS} --with-default-gems="${DEFAULT_GEMS}" 2>&1 +} + +install_rvm() { + mode=$1 + # Install RVM + receive_gpg_keys RVM_GPG_KEYS + # Determine appropriate settings for rvm installer + set_rvm_install_args $RUBY_VERSION + # Create rvm group as a system group to reduce the odds of conflict with local user UIDs + if ! cat /etc/group | grep -e "^rvm:" > /dev/null 2>&1; then + groupadd -r rvm + fi + # Install rvm + curl -sSL https://get.rvm.io | bash -s stable --ignore-dotfiles ${RVM_INSTALL_ARGS} --with-default-gems="${DEFAULT_GEMS}" 2>&1 || install_previous_version "$mode" + sudo usermod -aG rvm ${USERNAME} + source /usr/local/rvm/scripts/rvm + rvm fix-permissions system + rm -rf ${GNUPGHOME} +} + +install_rvm "mode1" +echo -e "\n๐Ÿ‘‰๐Ÿป๐Ÿ‘‰๐ŸปRVM version installed by test file ... (mode: 1 - install using find_prev_version_from_git_tags):" +check "rvm" rvm --version + +install_rvm "mode2" +echo -e "\n๐Ÿ‘‰๐Ÿป๐Ÿ‘‰๐ŸปRVM version installed by test file ... (mode: 1 - install using GitHub Api):" +check "rvm" rvm --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/ruby/scenarios.json b/test/ruby/scenarios.json index 04cac73f5..cfa2d8554 100644 --- a/test/ruby/scenarios.json +++ b/test/ruby/scenarios.json @@ -13,5 +13,13 @@ "features": { "ruby": {} } + }, + "ruby_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:bullseye", + "features": { + "ruby": { + "version": "latest" + } + } } } \ No newline at end of file From bb7b7ea29f84ea31262c1290a2e14e9984286295 Mon Sep 17 00:00:00 2001 From: Prathamesh Zarkar <159782310+prathameshzarkar9@users.noreply.github.com> Date: Tue, 23 Apr 2024 03:50:07 +0530 Subject: [PATCH 037/247] upgrade cuda version to 11.7 and cudnn to 8.5.0 (#942) upgrade cuda version 11.7 and cudnn 8.5.0 --- src/nvidia-cuda/devcontainer-feature.json | 2 +- src/nvidia-cuda/install.sh | 7 +++++++ test/nvidia-cuda/install_cudnn_nvxt_version.sh | 8 ++++---- test/nvidia-cuda/scenarios.json | 4 ++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/nvidia-cuda/devcontainer-feature.json b/src/nvidia-cuda/devcontainer-feature.json index bb63ae1c7..4a0fb0834 100644 --- a/src/nvidia-cuda/devcontainer-feature.json +++ b/src/nvidia-cuda/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "nvidia-cuda", - "version": "1.1.1", + "version": "1.1.2", "name": "NVIDIA CUDA", "description": "Installs shared libraries for NVIDIA CUDA.", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/nvidia-cuda", diff --git a/src/nvidia-cuda/install.sh b/src/nvidia-cuda/install.sh index d8658964e..79a7a9a20 100644 --- a/src/nvidia-cuda/install.sh +++ b/src/nvidia-cuda/install.sh @@ -12,6 +12,8 @@ INSTALL_TOOLKIT=${INSTALLTOOLKIT} CUDA_VERSION=${CUDAVERSION} CUDNN_VERSION=${CUDNNVERSION} +. /etc/os-release + if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' exit 1 @@ -33,6 +35,11 @@ check_packages() { fi } +if [ $VERSION_CODENAME = "bookworm" ] || [ $VERSION_CODENAME = "jammy" ] && [ $CUDA_VERSION \< 11.7 ]; then + echo "(!) Unsupported distribution version '${VERSION_CODENAME}' for CUDA < 11.7" + exit 1 +fi + export DEBIAN_FRONTEND=noninteractive check_packages wget ca-certificates diff --git a/test/nvidia-cuda/install_cudnn_nvxt_version.sh b/test/nvidia-cuda/install_cudnn_nvxt_version.sh index a7f46bdd6..4817ae479 100644 --- a/test/nvidia-cuda/install_cudnn_nvxt_version.sh +++ b/test/nvidia-cuda/install_cudnn_nvxt_version.sh @@ -5,11 +5,11 @@ set -e # Optional: Import test library source dev-container-features-test-lib -# Check installation of libcudnn8 (8.3.2) -check "libcudnn.so.8.3.2" test 1 -eq "$(find /usr -name 'libcudnn.so.8.3.2' | wc -l)" +# Check installation of libcudnn8 (8.5.0) +check "libcudnn.so.8.5.0" test 1 -eq "$(find /usr -name 'libcudnn.so.8.5.0' | wc -l)" -# Check installation of cuda-nvtx-11-5 (11.5) -check "cuda-11-5+nvtx" test -e '/usr/local/cuda-11.5/targets/x86_64-linux/include/nvtx3' +# Check installation of cuda-nvtx-11-7 (11.7) +check "cuda-11-7+nvtx" test -e '/usr/local/cuda-11.7/targets/x86_64-linux/include/nvtx3' # Report result reportResults diff --git a/test/nvidia-cuda/scenarios.json b/test/nvidia-cuda/scenarios.json index 3018330e2..bd73263b3 100644 --- a/test/nvidia-cuda/scenarios.json +++ b/test/nvidia-cuda/scenarios.json @@ -16,8 +16,8 @@ "nvidia-cuda": { "installCudnn": true, "installNvtx": true, - "cudaVersion": "11.5", - "cudnnVersion": "8.3.2.44" + "cudaVersion": "11.7", + "cudnnVersion": "8.5.0.96" } } } From d8e9d335952d0b8d488cc3445ad076003bc3c22c Mon Sep 17 00:00:00 2001 From: Barrett Strausser Date: Tue, 7 May 2024 11:10:39 -0400 Subject: [PATCH 038/247] Fix rustup-init sha256sum check (#962) * Fix rustup-init sha256sum check * Semver update --------- Co-authored-by: bstrausser --- src/rust/devcontainer-feature.json | 2 +- src/rust/install.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rust/devcontainer-feature.json b/src/rust/devcontainer-feature.json index c4a7dd5b9..70013442f 100644 --- a/src/rust/devcontainer-feature.json +++ b/src/rust/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "rust", - "version": "1.1.2", + "version": "1.1.3", "name": "Rust", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/rust", "description": "Installs Rust, common Rust utilities, and their required dependencies", diff --git a/src/rust/install.sh b/src/rust/install.sh index 00c0a6e72..4db9edc1e 100755 --- a/src/rust/install.sh +++ b/src/rust/install.sh @@ -186,6 +186,7 @@ else curl -sSL --proto '=https' --tlsv1.2 "https://static.rust-lang.org/rustup/dist/${download_architecture}-unknown-linux-gnu/rustup-init" -o /tmp/rustup/target/${download_architecture}-unknown-linux-gnu/release/rustup-init curl -sSL --proto '=https' --tlsv1.2 "https://static.rust-lang.org/rustup/dist/${download_architecture}-unknown-linux-gnu/rustup-init.sha256" -o /tmp/rustup/rustup-init.sha256 cd /tmp/rustup + cp /tmp/rustup/target/${download_architecture}-unknown-linux-gnu/release/rustup-init /tmp/rustup/rustup-init sha256sum -c rustup-init.sha256 chmod +x target/${download_architecture}-unknown-linux-gnu/release/rustup-init target/${download_architecture}-unknown-linux-gnu/release/rustup-init -y --no-modify-path --profile ${RUSTUP_PROFILE} ${default_toolchain_arg} From 67c10a660868260c284b02f2878dcfdcfd91279f Mon Sep 17 00:00:00 2001 From: Jacob Woffenden Date: Mon, 13 May 2024 17:37:11 +0100 Subject: [PATCH 039/247] =?UTF-8?q?=E2=9C=A8=20Add=20Ubuntu=2024=20Noble?= =?UTF-8?q?=20to=20`docker-in-docker`=20(#971)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Ubuntu Noble Signed-off-by: GitHub * Changes Signed-off-by: GitHub --------- Signed-off-by: GitHub --- .github/workflows/test-all.yaml | 1 + .github/workflows/test-pr.yaml | 1 + src/docker-in-docker/devcontainer-feature.json | 2 +- src/docker-in-docker/install.sh | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-all.yaml b/.github/workflows/test-all.yaml index 40f5efa75..7a9554314 100644 --- a/.github/workflows/test-all.yaml +++ b/.github/workflows/test-all.yaml @@ -48,6 +48,7 @@ jobs: "debian:12", "mcr.microsoft.com/devcontainers/base:ubuntu", "mcr.microsoft.com/devcontainers/base:debian", + "mcr.microsoft.com/devcontainers/base:noble" ] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index 3aae841e2..776a9731f 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -55,6 +55,7 @@ jobs: "debian:12", "mcr.microsoft.com/devcontainers/base:ubuntu", "mcr.microsoft.com/devcontainers/base:debian", + "mcr.microsoft.com/devcontainers/base:noble" ] steps: - uses: actions/checkout@v3 diff --git a/src/docker-in-docker/devcontainer-feature.json b/src/docker-in-docker/devcontainer-feature.json index 812db444c..4897ebf3e 100644 --- a/src/docker-in-docker/devcontainer-feature.json +++ b/src/docker-in-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-in-docker", - "version": "2.10.2", + "version": "2.11.0", "name": "Docker (Docker-in-Docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-in-docker", "description": "Create child containers *inside* a container, independent from the host's docker instance. Installs Docker extension in the container along with needed CLIs.", diff --git a/src/docker-in-docker/install.sh b/src/docker-in-docker/install.sh index 0dc9e52d1..ee9cb6ee6 100755 --- a/src/docker-in-docker/install.sh +++ b/src/docker-in-docker/install.sh @@ -18,8 +18,8 @@ USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" INSTALL_DOCKER_BUILDX="${INSTALLDOCKERBUILDX:-"true"}" INSTALL_DOCKER_COMPOSE_SWITCH="${INSTALLDOCKERCOMPOSESWITCH:-"true"}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" -DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal jammy" -DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal hirsute impish jammy" +DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal jammy noble" +DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal hirsute impish jammy noble" # Default: Exit on any failure. set -e From 4d2dabec57722e23ffe547038923a799e332e291 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Fri, 17 May 2024 00:12:03 +0530 Subject: [PATCH 040/247] [Desktop-lite]- libasound2 not installing in noble - issue (#973) * [Desktop-lite]- libasound2 not installing in noble - issue * bump to patch version in Desktop-lite feature * misc change * Changes for comments ( review comments ) * changes based on review comments.. --- src/desktop-lite/devcontainer-feature.json | 2 +- src/desktop-lite/install.sh | 11 +++++++- test/desktop-lite/test.sh | 30 ++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/desktop-lite/devcontainer-feature.json b/src/desktop-lite/devcontainer-feature.json index 5387138b5..5386eb31d 100644 --- a/src/desktop-lite/devcontainer-feature.json +++ b/src/desktop-lite/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "desktop-lite", - "version": "1.0.8", + "version": "1.1.0", "name": "Light-weight Desktop", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/desktop-lite", "description": "Adds a lightweight Fluxbox based desktop to the container that can be accessed using a VNC viewer or the web. GUI-based commands executed from the built-in VS code terminal will open on the desktop automatically.", diff --git a/src/desktop-lite/install.sh b/src/desktop-lite/install.sh index df4390eca..13a524ada 100755 --- a/src/desktop-lite/install.sh +++ b/src/desktop-lite/install.sh @@ -41,7 +41,6 @@ package_list=" libnotify4 \ libnss3 \ libxss1 \ - libasound2 \ xfonts-base \ xfonts-terminus \ fonts-noto \ @@ -198,6 +197,16 @@ fi # Install X11, fluxbox and VS Code dependencies check_packages ${package_list} +# if Ubuntu-24.04, noble(numbat) found, then will install libasound2-dev instead of libasound2. +# this change is temporary, https://packages.ubuntu.com/noble/libasound2 will switch to libasound2 once it is available for Ubuntu-24.04, noble(numbat) +. /etc/os-release +if [ "${ID}" = "ubuntu" ] && [ "${VERSION_CODENAME}" = "noble" ]; then + echo "Ubuntu 24.04, Noble(Numbat) detected. Installing libasound2-dev package..." + check_packages "libasound2-dev" +else + check_packages "libasound2" +fi + # On newer versions of Ubuntu (22.04), # we need an additional package that isn't provided in earlier versions if ! type vncpasswd > /dev/null 2>&1; then diff --git a/test/desktop-lite/test.sh b/test/desktop-lite/test.sh index 9009aa9c6..5d11dd424 100755 --- a/test/desktop-lite/test.sh +++ b/test/desktop-lite/test.sh @@ -5,9 +5,39 @@ set -e # Optional: Import test library source dev-container-features-test-lib +echoStderr() +{ + echo "$@" 1>&2 +} + +checkOSPackage() { + LABEL=$1 + PACKAGE_NAME=$2 + echo -e "\n๐Ÿงช Testing $LABEL" + # Check if the package exists and retrieve its exact version + if [ "$(dpkg-query -W -f='${Status}' "$PACKAGE_NAME" 2>/dev/null | grep -c "ok installed")" -eq 1 ]; then + echo "โœ… Package '$PACKAGE_NAME' is installed." + exit 0 + else + echo "โŒ Package '$PACKAGE_NAME' is not installed." + exit 1 + fi +} + check "desktop-init-exists" bash -c "ls /usr/local/share/desktop-init.sh" check "log-exists" bash -c "ls /tmp/container-init.log" check "fluxbox-exists" bash -c "ls -la ~/.fluxbox" +. /etc/os-release +if [ "${ID}" = "ubuntu" ]; then + if [ "${VERSION_CODENAME}" = "noble" ]; then + checkOSPackage "if libasound2-dev exists !" "libasound2-dev" + else + checkOSPackage "if libasound2 exists !" "libasound2" + fi +else + checkOSPackage "if libasound2 exists !" "libasound2" +fi + # Report result reportResults \ No newline at end of file From ecbfd50952e513db872d8d3380e069ccf74a70a8 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Fri, 17 May 2024 09:16:22 -0700 Subject: [PATCH 041/247] [Updates] Automated vendor dotnet-install script (#970) * Automated dotnet-install script update * Bump version --------- Co-authored-by: github-actions --- src/dotnet/devcontainer-feature.json | 2 +- src/dotnet/scripts/vendor/dotnet-install.sh | 26 +++++++++++++++++---- src/oryx/devcontainer-feature.json | 2 +- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/dotnet/devcontainer-feature.json b/src/dotnet/devcontainer-feature.json index 78a061d23..8b8ffaf2a 100644 --- a/src/dotnet/devcontainer-feature.json +++ b/src/dotnet/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "dotnet", - "version": "2.0.5", + "version": "2.0.6", "name": "Dotnet CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/dotnet", "description": "This Feature installs the latest .NET SDK, which includes the .NET CLI and the shared runtime. Options are provided to choose a different version or additional versions.", diff --git a/src/dotnet/scripts/vendor/dotnet-install.sh b/src/dotnet/scripts/vendor/dotnet-install.sh index f6b08d1f8..42c201af4 100755 --- a/src/dotnet/scripts/vendor/dotnet-install.sh +++ b/src/dotnet/scripts/vendor/dotnet-install.sh @@ -298,6 +298,10 @@ get_machine_architecture() { if command -v uname > /dev/null; then CPUName=$(uname -m) case $CPUName in + armv1*|armv2*|armv3*|armv4*|armv5*|armv6*) + echo "armv6-or-below" + return 0 + ;; armv*l) echo "arm" return 0 @@ -339,7 +343,13 @@ get_normalized_architecture_from_architecture() { local architecture="$(to_lowercase "$1")" if [[ $architecture == \ ]]; then - echo "$(get_machine_architecture)" + machine_architecture="$(get_machine_architecture)" + if [[ "$machine_architecture" == "armv6-or-below" ]]; then + say_err "Architecture \`$machine_architecture\` not supported. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues" + return 1 + fi + + echo $machine_architecture return 0 fi @@ -1013,7 +1023,7 @@ extract_dotnet_package() { rm -rf "$temp_out_path" if [ -z ${keep_zip+x} ]; then - rm -f "$zip_path" && say_verbose "Temporary zip file $zip_path was removed" + rm -f "$zip_path" && say_verbose "Temporary archive file $zip_path was removed" fi if [ "$failed" = true ]; then @@ -1261,6 +1271,12 @@ get_download_link_from_aka_ms() { http_codes=$( echo "$response" | awk '$1 ~ /^HTTP/ {print $2}' ) # They all need to be 301, otherwise some links are broken (except for the last, which is not a redirect but 200 or 404). broken_redirects=$( echo "$http_codes" | sed '$d' | grep -v '301' ) + # The response may end without final code 2xx/4xx/5xx somehow, e.g. network restrictions on www.bing.com causes redirecting to bing.com fails with connection refused. + # In this case it should not exclude the last. + last_http_code=$( echo "$http_codes" | tail -n 1 ) + if ! [[ $last_http_code =~ ^(2|4|5)[0-9][0-9]$ ]]; then + broken_redirects=$( echo "$http_codes" | grep -v '301' ) + fi # All HTTP codes are 301 (Moved Permanently), the redirect link exists. if [[ -z "$broken_redirects" ]]; then @@ -1512,7 +1528,7 @@ install_dotnet() { mkdir -p "$install_root" zip_path="${zip_path:-$(mktemp "$temporary_file_template")}" - say_verbose "Zip path: $zip_path" + say_verbose "Archive path: $zip_path" for link_index in "${!download_links[@]}" do @@ -1536,7 +1552,7 @@ install_dotnet() { say "Failed to download $link_type link '$download_link': $download_error_msg" ;; esac - rm -f "$zip_path" 2>&1 && say_verbose "Temporary zip file $zip_path was removed" + rm -f "$zip_path" 2>&1 && say_verbose "Temporary archive file $zip_path was removed" else download_completed=true break @@ -1551,7 +1567,7 @@ install_dotnet() { remote_file_size="$(get_remote_file_size "$download_link")" - say "Extracting zip from $download_link" + say "Extracting archive from $download_link" extract_dotnet_package "$zip_path" "$install_root" "$remote_file_size" || return 1 # Check if the SDK version is installed; if not, fail the installation. diff --git a/src/oryx/devcontainer-feature.json b/src/oryx/devcontainer-feature.json index 41bd1f759..9e6e698a1 100644 --- a/src/oryx/devcontainer-feature.json +++ b/src/oryx/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "oryx", - "version": "1.3.2", + "version": "1.3.3", "name": "Oryx", "description": "Installs the oryx CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/oryx", From f5787eed01022f177475a99084327e023a84ddaf Mon Sep 17 00:00:00 2001 From: Andy Li Date: Wed, 22 May 2024 00:57:40 +0100 Subject: [PATCH 042/247] Add Ubuntu 24 Noble to `docker-outside-of-docker` (#978) --- src/docker-outside-of-docker/devcontainer-feature.json | 2 +- src/docker-outside-of-docker/install.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/docker-outside-of-docker/devcontainer-feature.json b/src/docker-outside-of-docker/devcontainer-feature.json index 2506031ae..19018ea8e 100644 --- a/src/docker-outside-of-docker/devcontainer-feature.json +++ b/src/docker-outside-of-docker/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "docker-outside-of-docker", - "version": "1.4.5", + "version": "1.5.0", "name": "Docker (docker-outside-of-docker)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker", "description": "Re-use the host docker socket, adding the Docker CLI to a container. Feature invokes a script to enable using a forwarded Docker socket within a container to run Docker commands.", diff --git a/src/docker-outside-of-docker/install.sh b/src/docker-outside-of-docker/install.sh index 16ad15f60..de5212fe6 100755 --- a/src/docker-outside-of-docker/install.sh +++ b/src/docker-outside-of-docker/install.sh @@ -19,8 +19,8 @@ USERNAME="${USERNAME:-"${_REMOTE_USER:-"automatic"}"}" INSTALL_DOCKER_BUILDX="${INSTALLDOCKERBUILDX:-"true"}" MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" -DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal jammy" -DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal hirsute impish jammy" +DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal jammy noble" +DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="bookworm buster bullseye bionic focal hirsute impish jammy noble" set -e From 02b71cbd6cf972ca29059cd409dc0fe8c3b60e65 Mon Sep 17 00:00:00 2001 From: hellodword <46193371+hellodword@users.noreply.github.com> Date: Tue, 28 May 2024 23:03:23 +0000 Subject: [PATCH 043/247] [python] add default formatter (#903) Co-authored-by: Samruddhi Khandale --- src/python/devcontainer-feature.json | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 9f211244f..ef16ee643 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.4.2", + "version": "1.5.0", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.", @@ -73,10 +73,14 @@ "vscode": { "extensions": [ "ms-python.python", - "ms-python.vscode-pylance" + "ms-python.vscode-pylance", + "ms-python.autopep8" ], "settings": { - "python.defaultInterpreterPath": "/usr/local/python/current/bin/python" + "python.defaultInterpreterPath": "/usr/local/python/current/bin/python", + "[python]": { + "editor.defaultFormatter": "ms-python.autopep8" + } } } }, @@ -84,4 +88,4 @@ "ghcr.io/devcontainers/features/common-utils", "ghcr.io/devcontainers/features/oryx" ] -} \ No newline at end of file +} From 10ea0b7dd5a653b08266039527f7e095c02591a8 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Tue, 28 May 2024 16:20:00 -0700 Subject: [PATCH 044/247] Automated documentation update (#984) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/python/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/README.md b/src/python/README.md index db6ad1039..90d79aee8 100644 --- a/src/python/README.md +++ b/src/python/README.md @@ -31,6 +31,7 @@ Installs the provided version of Python, as well as PIPX, and other common Pytho - `ms-python.python` - `ms-python.vscode-pylance` +- `ms-python.autopep8` From 476a68d0523b004112498dc161c2b6de1bd9fe57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Wed, 29 May 2024 21:43:03 +0200 Subject: [PATCH 045/247] [desktop-lite] Allow password-less VNC connections (#982) * [desktop-lite] Allow password-less VNC connections Closes #611 * Restore readme * Update src/desktop-lite/install.sh Co-authored-by: Samruddhi Khandale * Fix --------- Co-authored-by: Samruddhi Khandale --- src/desktop-lite/devcontainer-feature.json | 13 +++++----- src/desktop-lite/install.sh | 29 +++++++++++++++++++--- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/desktop-lite/devcontainer-feature.json b/src/desktop-lite/devcontainer-feature.json index 5386eb31d..417787c51 100644 --- a/src/desktop-lite/devcontainer-feature.json +++ b/src/desktop-lite/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "desktop-lite", - "version": "1.1.0", + "version": "1.2.0", "name": "Light-weight Desktop", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/desktop-lite", "description": "Adds a lightweight Fluxbox based desktop to the container that can be accessed using a VNC viewer or the web. GUI-based commands executed from the built-in VS code terminal will open on the desktop automatically.", @@ -19,17 +19,18 @@ "1.2.0" ], "default": "1.2.0", - "description": "NoVnc Version" + "description": "The noVNC version to use" }, "password": { "type": "string", "proposals": [ "vscode", "codespaces", - "password" + "password", + "noPassword" ], "default": "vscode", - "description": "Enter a password for desktop connections" + "description": "Enter a password for desktop connections. If \"noPassword\", connections from the local host can be established without entering a password" }, "webPort": { "type": "string", @@ -37,7 +38,7 @@ "6080" ], "default": "6080", - "description": "Enter a port for the VNC web client" + "description": "Enter a port for the VNC web client (noVNC)" }, "vncPort": { "type": "string", @@ -45,7 +46,7 @@ "5901" ], "default": "5901", - "description": "Enter a port for the desktop VNC server" + "description": "Enter a port for the desktop VNC server (TigerVNC)" } }, "init": true, diff --git a/src/desktop-lite/install.sh b/src/desktop-lite/install.sh index 13a524ada..ef8b603c6 100755 --- a/src/desktop-lite/install.sh +++ b/src/desktop-lite/install.sh @@ -9,6 +9,9 @@ NOVNC_VERSION="${NOVNCVERSION:-"1.2.0"}" # TODO: Add in a 'latest' auto-detect and swap name to 'version' VNC_PASSWORD=${PASSWORD:-"vscode"} +if [ "$VNC_PASSWORD" = "noPassword" ]; then + unset VNC_PASSWORD +fi NOVNC_PORT="${WEBPORT:-6080}" VNC_PORT="${VNCPORT:-5901}" @@ -372,7 +375,15 @@ sudoIf chown root:\${group_name} /tmp/.X11-unix if [ "\$(echo "\${VNC_RESOLUTION}" | tr -cd 'x' | wc -c)" = "1" ]; then VNC_RESOLUTION=\${VNC_RESOLUTION}x16; fi screen_geometry="\${VNC_RESOLUTION%*x*}" screen_depth="\${VNC_RESOLUTION##*x}" -startInBackgroundIfNotRunning "Xtigervnc" sudoUserIf "tigervncserver \${DISPLAY} -geometry \${screen_geometry} -depth \${screen_depth} -rfbport ${VNC_PORT} -dpi \${VNC_DPI:-96} -localhost -desktop fluxbox -fg -passwd /usr/local/etc/vscode-dev-containers/vnc-passwd" + +# Check if VNC_PASSWORD is set and use the appropriate command +common_options="tigervncserver \${DISPLAY} -geometry \${screen_geometry} -depth \${screen_depth} -rfbport ${VNC_PORT} -dpi \${VNC_DPI:-96} -localhost -desktop fluxbox -fg" + +if [ -n "\${VNC_PASSWORD+x}" ]; then + startInBackgroundIfNotRunning "Xtigervnc" sudoUserIf "\${common_options} -passwd /usr/local/etc/vscode-dev-containers/vnc-passwd" +else + startInBackgroundIfNotRunning "Xtigervnc" sudoUserIf "\${common_options} -SecurityTypes None" +fi # Spin up noVNC if installed and not running. if [ -d "/usr/local/novnc" ] && [ "\$(ps -ef | grep /usr/local/novnc/noVNC*/utils/launch.sh | grep -v grep)" = "" ]; then @@ -388,7 +399,9 @@ exec "\$@" log "** SCRIPT EXIT **" EOF -echo "${VNC_PASSWORD}" | vncpasswd -f > /usr/local/etc/vscode-dev-containers/vnc-passwd +if [ -n "${VNC_PASSWORD+x}" ]; then + echo "${VNC_PASSWORD}" | vncpasswd -f > /usr/local/etc/vscode-dev-containers/vnc-passwd +fi chmod +x /usr/local/share/desktop-init.sh /usr/local/bin/set-resolution # Set up fluxbox config @@ -401,15 +414,23 @@ fi # Clean up rm -rf /var/lib/apt/lists/* +# Determine the message based on whether VNC_PASSWORD is set +if [ -n "${VNC_PASSWORD+x}" ]; then + PASSWORD_MESSAGE="In both cases, use the password \"${VNC_PASSWORD}\" when connecting" +else + PASSWORD_MESSAGE="In both cases, no password is required." +fi + +# Display the message cat << EOF You now have a working desktop! Connect to in one of the following ways: -- Forward port ${NOVNC_PORT} and use a web browser start the noVNC client (recommended) +- Forward port ${NOVNC_PORT} and use a web browser to start the noVNC client (recommended) - Forward port ${VNC_PORT} using VS Code client and connect using a VNC Viewer -In both cases, use the password "${VNC_PASSWORD}" when connecting +${PASSWORD_MESSAGE} (*) Done! From 32797f4f693a3bf18a18928ad28bc55589a7ada6 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Wed, 29 May 2024 12:54:52 -0700 Subject: [PATCH 046/247] Automated documentation update (#989) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/desktop-lite/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/desktop-lite/README.md b/src/desktop-lite/README.md index 7094a2425..6f2d67ce2 100644 --- a/src/desktop-lite/README.md +++ b/src/desktop-lite/README.md @@ -16,10 +16,10 @@ Adds a lightweight Fluxbox based desktop to the container that can be accessed u | Options Id | Description | Type | Default Value | |-----|-----|-----|-----| | version | Currently Unused! | string | latest | -| noVncVersion | NoVnc Version | string | 1.2.0 | -| password | Enter a password for desktop connections | string | vscode | -| webPort | Enter a port for the VNC web client | string | 6080 | -| vncPort | Enter a port for the desktop VNC server | string | 5901 | +| noVncVersion | The noVNC version to use | string | 1.2.0 | +| password | Enter a password for desktop connections. If "noPassword", connections from the local host can be established without entering a password | string | vscode | +| webPort | Enter a port for the VNC web client (noVNC) | string | 6080 | +| vncPort | Enter a port for the desktop VNC server (TigerVNC) | string | 5901 | ## Connecting to the desktop From c1df45b189afae33be72c1ce478452fbfc044abe Mon Sep 17 00:00:00 2001 From: Prabhakar Kumar <64955767+prabhakk-mw@users.noreply.github.com> Date: Thu, 30 May 2024 03:00:18 +0530 Subject: [PATCH 047/247] Adds /home/USER/.local/bin/ to PATH in /etc/sudoers.d/vscode, (#887) * Adds /home/USER/.local/bin/ to PATH in /etc/sudoers.d/vscode, fixes devcontainers/features#870 * Bumping up version to 1.4.2 * Adds to sudoers file if already present * Tests to ensure Default secure_path is not overwritten * Bump to version 1.4.4 * Fix version as 1.4.3 Co-authored-by: Samruddhi Khandale * Update src/python/install.sh Co-authored-by: Prabhakar Kumar <64955767+prabhakk-mw@users.noreply.github.com> * Update src/python/devcontainer-feature.json --------- Co-authored-by: Samruddhi Khandale Co-authored-by: Samruddhi Khandale --- src/python/devcontainer-feature.json | 2 +- src/python/install.sh | 57 ++++++++++++------- test/python/install_jupyterlab.sh | 3 + ...nstall_jupyterlab_existing_sudoers_file.sh | 36 ++++++++++++ .../Dockerfile | 5 ++ .../sudoers.test | 2 + test/python/scenarios.json | 13 +++++ 7 files changed, 97 insertions(+), 21 deletions(-) create mode 100755 test/python/install_jupyterlab_existing_sudoers_file.sh create mode 100644 test/python/install_jupyterlab_existing_sudoers_file/Dockerfile create mode 100644 test/python/install_jupyterlab_existing_sudoers_file/sudoers.test diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index ef16ee643..7c2c6200a 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.5.0", + "version": "1.6.0", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.", diff --git a/src/python/install.sh b/src/python/install.sh index e8a9d24e1..1aad3f631 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -130,7 +130,7 @@ updaterc() { fi } -# Import the specified key in a variable name passed in as +# Import the specified key in a variable name passed in as receive_gpg_keys() { local keys=${!1} local keyring_args="" @@ -152,7 +152,7 @@ receive_gpg_keys() { local retry_count=0 local gpg_ok="false" set +e - until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; do echo "(*) Downloading GPG key..." ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" @@ -222,7 +222,7 @@ find_version_from_git_tags() { local repository=$2 local prefix=${3:-"tags/v"} local separator=${4:-"."} - local last_part_optional=${5:-"false"} + local last_part_optional=${5:-"false"} if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then local escaped_separator=${separator//./\\.} local last_part @@ -282,7 +282,7 @@ find_prev_version_from_git_tags() { ((breakfix=breakfix-1)) if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then declare -g ${variable_name}="${major}.${minor}" - else + else declare -g ${variable_name}="${major}.${minor}.${breakfix}" fi fi @@ -378,13 +378,13 @@ check_packages() { add_symlink() { if [[ ! -d "${CURRENT_PATH}" ]]; then - ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" + ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" fi if [ "${OVERRIDE_DEFAULT_VERSION}" = "true" ]; then if [[ $(ls -l ${CURRENT_PATH}) != *"-> ${INSTALL_PATH}"* ]] ; then rm "${CURRENT_PATH}" - ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" + ln -s -r "${INSTALL_PATH}" "${CURRENT_PATH}" fi fi } @@ -397,7 +397,7 @@ install_openssl3() { openssl3_version="3.0" # Find version using soft match find_version_from_git_tags openssl3_version "https://github.com/openssl/openssl" "openssl-" - local tgz_filename="openssl-${openssl3_version}.tar.gz" + local tgz_filename="openssl-${openssl3_version}.tar.gz" local tgz_url="https://github.com/openssl/openssl/releases/download/openssl-${openssl3_version}/${tgz_filename}" echo "Downloading ${tgz_filename}..." curl -sSL -o "/tmp/openssl3/${tgz_filename}" "${tgz_url}" @@ -434,7 +434,7 @@ install_cpython() { } install_from_source() { - VERSION=$1 + VERSION=$1 echo "(*) Building Python ${VERSION} from source..." if ! type git > /dev/null 2>&1; then check_packages git @@ -444,7 +444,7 @@ install_from_source() { find_version_from_git_tags VERSION "https://github.com/python/cpython" # Some platforms/os versions need modern versions of openssl installed - # via common package repositories, for now rhel-7 family, use case statement to + # via common package repositories, for now rhel-7 family, use case statement to # make it easy to expand case ${VERSION_CODENAME} in centos7|rhel7) @@ -455,7 +455,7 @@ install_from_source() { esac install_cpython "${VERSION}" - if [ -f "/tmp/python-src/${cpython_tgz_filename}" ]; then + if [ -f "/tmp/python-src/${cpython_tgz_filename}" ]; then if grep -q "404 Not Found" "/tmp/python-src/${cpython_tgz_filename}"; then install_prev_vers_cpython "${VERSION}" fi @@ -512,9 +512,9 @@ install_from_source() { } install_using_oryx() { - VERSION=$1 + VERSION=$1 INSTALL_PATH="${PYTHON_INSTALL_PATH}/${VERSION}" - + if [ -d "${INSTALL_PATH}" ]; then echo "(!) Python version ${VERSION} already exists." exit 1 @@ -727,7 +727,7 @@ if [ "${PYTHON_VERSION}" != "none" ]; then usermod -a -G python "${USERNAME}" CURRENT_PATH="${PYTHON_INSTALL_PATH}/current" - + install_python ${PYTHON_VERSION} # Additional python versions to be installed but not be set as default. @@ -748,7 +748,7 @@ if [ "${PYTHON_VERSION}" != "none" ]; then updaterc "if [[ \"\${PATH}\" != *\"${CURRENT_PATH}/bin\"* ]]; then export PATH=${CURRENT_PATH}/bin:\${PATH}; fi" PATH="${INSTALL_PATH}/bin:${PATH}" fi - + # Updates the symlinks for os-provided, or the installed python version in other cases chown -R "${USERNAME}:python" "${PYTHON_INSTALL_PATH}" chmod -R g+r+w "${PYTHON_INSTALL_PATH}" @@ -776,7 +776,7 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then umask 0002 mkdir -p ${PIPX_BIN_DIR} chown -R "${USERNAME}:pipx" ${PIPX_HOME} - chmod -R g+r+w "${PIPX_HOME}" + chmod -R g+r+w "${PIPX_HOME}" find "${PIPX_HOME}" -type d -print0 | xargs -0 -n 1 chmod g+s # Update pip if not using os provided python @@ -805,21 +805,21 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then echo "${util} already installed. Skipping." fi done - + # Temporary: Removes โ€œsetup toolsโ€ metadata directory due to https://github.com/advisories/GHSA-r9hx-vwmv-q579 - if [[ $SKIP_VULNERABILITY_PATCHING = "false" ]]; then + if [[ $SKIP_VULNERABILITY_PATCHING = "false" ]]; then VULNERABLE_VERSIONS=("3.10" "3.11") RUN_TIME_PY_VER_DETECT=$(${PYTHON_SRC} --version 2>&1) PY_MAJOR_MINOR_VER=${RUN_TIME_PY_VER_DETECT:7:4}; if [[ ${VULNERABLE_VERSIONS[*]} =~ $PY_MAJOR_MINOR_VER ]]; then rm -rf ${PIPX_HOME}/shared/lib/"python${PY_MAJOR_MINOR_VER}"/site-packages/setuptools-65.5.0.dist-info - if [[ -e "/usr/local/lib/python${PY_MAJOR_MINOR_VER}/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl" ]]; then + if [[ -e "/usr/local/lib/python${PY_MAJOR_MINOR_VER}/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl" ]]; then # remove the vulnerable setuptools-65.5.0-py3-none-any.whl file rm /usr/local/lib/python${PY_MAJOR_MINOR_VER}/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl # create and change to the setuptools_downloaded directory mkdir -p /tmp/setuptools_downloaded cd /tmp/setuptools_downloaded - # download the source distribution for setuptools using pip + # download the source distribution for setuptools using pip pip download setuptools==65.5.1 --no-binary :all: # extract the filename of the setuptools-*.tar.gz file filename=$(find . -maxdepth 1 -type f) @@ -833,7 +833,7 @@ if [[ "${INSTALL_PYTHON_TOOLS}" = "true" ]] && [[ -n "${PYTHON_SRC}" ]]; then python setup.py bdist_wheel # move inside the dist directory in pwd cd dist - # copy this file to the ensurepip/_bundled directory + # copy this file to the ensurepip/_bundled directory cp setuptools-65.5.1-py3-none-any.whl /usr/local/lib/python${PY_MAJOR_MINOR_VER}/ensurepip/_bundled/ # replace the version in __init__.py file with the installed version sed -i 's/_SETUPTOOLS_VERSION = \"65\.5\.0\"/_SETUPTOOLS_VERSION = "65.5.1"/g' /usr/local/lib/"python${PY_MAJOR_MINOR_VER}"/ensurepip/__init__.py @@ -865,6 +865,23 @@ if [ "${INSTALL_JUPYTERLAB}" = "true" ]; then install_user_package $INSTALL_UNDER_ROOT jupyterlab install_user_package $INSTALL_UNDER_ROOT jupyterlab-git + if [ "$INSTALL_UNDER_ROOT" = false ]; then + # JupyterLab would have installed into /home/${USERNAME}/.local/bin + # Adding it to default path for Codespaces which use non-login shells + SUDOERS_FILE="/etc/sudoers.d/$USERNAME" + SEARCH_STR="Defaults secure_path=" + REPLACE_STR="Defaults secure_path=/home/${USERNAME}/.local/bin" + + if grep -qs ${SEARCH_STR} ${SUDOERS_FILE}; then + # string found and file is present + sed -i "s|${SEARCH_STR}|${REPLACE_STR}:|g" "${SUDOERS_FILE}" + else + # either string is not found, or file is not present + # In either case take same action, note >> places at end of file + echo "${REPLACE_STR}:${PATH}" >> ${SUDOERS_FILE} + fi + fi + # Configure JupyterLab if needed if [ -n "${CONFIGURE_JUPYTERLAB_ALLOW_ORIGIN}" ]; then # Resolve config directory diff --git a/test/python/install_jupyterlab.sh b/test/python/install_jupyterlab.sh index 58c4f7f54..033b44edd 100755 --- a/test/python/install_jupyterlab.sh +++ b/test/python/install_jupyterlab.sh @@ -22,5 +22,8 @@ check "jupyterlab_git" grep jupyterlab_git <<< "$packages" # Check for correct JupyterLab configuration check "config" grep ".*.allow_origin = '*'" /home/vscode/.jupyter/jupyter_server_config.py +# Check for PATH modification +check "default path has jupyterlab" sudo grep "/home/${user}/.local/bin" /etc/sudoers.d/$user + # Report result reportResults diff --git a/test/python/install_jupyterlab_existing_sudoers_file.sh b/test/python/install_jupyterlab_existing_sudoers_file.sh new file mode 100755 index 000000000..22bbc2f92 --- /dev/null +++ b/test/python/install_jupyterlab_existing_sudoers_file.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +# Always run these checks as the non-root user +user="$(whoami)" +check "user" grep vscode <<< "$user" + +# Check for an installation of JupyterLab +check "version" jupyter lab --version + +# Check location of JupyterLab installation +packages="$(python3 -m pip list)" +check "location" grep jupyter <<< "$packages" + +# Check for git extension +check "jupyterlab_git" grep jupyterlab_git <<< "$packages" + +# Check for correct JupyterLab configuration +check "config" grep ".*.allow_origin = '*'" /home/vscode/.jupyter/jupyter_server_config.py + +# Check for PATH modification +check "default path has jupyterlab" grep "Defaults secure_path=/home/${user}/.local/bin" /etc/sudoers.d/$user + +# Check if previous PATH exists +check "existing default path is preserved" grep "Defaults secure_path=.*original_content_of_sudoers_file" /etc/sudoers.d/$user + +# Check if PATH modification includes original and new paths +check "existing path included with jupyterlab" grep "Defaults secure_path.*/home/${user}/.local/bin.*original_content_of_sudoers_file" /etc/sudoers.d/$user + + +# Report result +reportResults diff --git a/test/python/install_jupyterlab_existing_sudoers_file/Dockerfile b/test/python/install_jupyterlab_existing_sudoers_file/Dockerfile new file mode 100644 index 000000000..5acb58a37 --- /dev/null +++ b/test/python/install_jupyterlab_existing_sudoers_file/Dockerfile @@ -0,0 +1,5 @@ +# Builds an image with a preconfigured SUDOERS file +# Used to test the install script for JupyterLab which modifies this file +FROM mcr.microsoft.com/devcontainers/base:focal + +COPY --chown=root sudoers.test /etc/sudoers.d/vscode diff --git a/test/python/install_jupyterlab_existing_sudoers_file/sudoers.test b/test/python/install_jupyterlab_existing_sudoers_file/sudoers.test new file mode 100644 index 000000000..9a26d777b --- /dev/null +++ b/test/python/install_jupyterlab_existing_sudoers_file/sudoers.test @@ -0,0 +1,2 @@ +# Sudoers File for testing, after install script runs the Defaults secure_path should be appended to +Defaults secure_path=/original_content_of_sudoers_file \ No newline at end of file diff --git a/test/python/scenarios.json b/test/python/scenarios.json index be37869df..e4fe0f0cd 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -68,6 +68,19 @@ } } }, + "install_jupyterlab_existing_sudoers_file": { + "build": { + "dockerfile": "Dockerfile" + }, + "remoteUser": "vscode", + "features": { + "python": { + "version": "latest", + "installJupyterlab": true, + "configureJupyterlabAllowOrigin": "*" + } + } + }, "install_jupyterlab_rhel_family": { "image": "almalinux:8", "remoteUser": "vscode", From b32aa5f0f207e9dc9c5ad36524ed45c6aaec20dd Mon Sep 17 00:00:00 2001 From: Rambaud Pierrick <12rambau@users.noreply.github.com> Date: Thu, 30 May 2024 19:56:16 +0200 Subject: [PATCH 048/247] fix centos-7 build (#985) * fix centos-7 build * Update src/python/install.sh Co-authored-by: Samruddhi Khandale * Update src/python/install.sh Co-authored-by: Samruddhi Khandale * Update src/python/install.sh Co-authored-by: Samruddhi Khandale * bump(python): 1.5.0 -> 1.5.1 --------- Co-authored-by: Samruddhi Khandale --- src/python/devcontainer-feature.json | 2 +- src/python/install.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index 7c2c6200a..f57568cc6 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.6.0", + "version": "1.6.1", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.", diff --git a/src/python/install.sh b/src/python/install.sh index 1aad3f631..94007a9b0 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -390,7 +390,6 @@ add_symlink() { } install_openssl3() { - local _prefix=$1 mkdir /tmp/openssl3 ( cd /tmp/openssl3 @@ -403,7 +402,7 @@ install_openssl3() { curl -sSL -o "/tmp/openssl3/${tgz_filename}" "${tgz_url}" tar xzf ${tgz_filename} cd openssl-${openssl3_version} - ./config --prefix=${_prefix} --openssldir=${_prefix} --libdir=lib + ./config --libdir=lib make -j $(nproc) make install_dev ) @@ -446,11 +445,12 @@ install_from_source() { # Some platforms/os versions need modern versions of openssl installed # via common package repositories, for now rhel-7 family, use case statement to # make it easy to expand + SSL_INSTALL_PATH="/usr/local" case ${VERSION_CODENAME} in centos7|rhel7) check_packages perl-IPC-Cmd - install_openssl3 ${INSTALL_PATH} - ADDL_CONFIG_ARGS="--with-openssl=${INSTALL_PATH} --with-openssl-rpath=${INSTALL_PATH}/lib" + install_openssl3 + ADDL_CONFIG_ARGS="--with-openssl=${SSL_INSTALL_PATH} --with-openssl-rpath=${SSL_INSTALL_PATH}/lib" ;; esac From dbb135408311512248bfb2b161d52d12936bbaa6 Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Fri, 31 May 2024 05:36:41 +0530 Subject: [PATCH 049/247] [azure-cli] - add support for noble numbat (#986) * [azure-cli] - add support for noble numbat * changes requested --- src/azure-cli/devcontainer-feature.json | 2 +- src/azure-cli/install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/azure-cli/devcontainer-feature.json b/src/azure-cli/devcontainer-feature.json index 6b26ef6fc..e73d2295c 100644 --- a/src/azure-cli/devcontainer-feature.json +++ b/src/azure-cli/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "azure-cli", - "version": "1.2.4", + "version": "1.2.5", "name": "Azure CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/azure-cli", "description": "Installs the Azure CLI along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", diff --git a/src/azure-cli/install.sh b/src/azure-cli/install.sh index a1b254779..2b52ca158 100755 --- a/src/azure-cli/install.sh +++ b/src/azure-cli/install.sh @@ -18,7 +18,7 @@ AZ_INSTALLBICEP=${INSTALLBICEP:-false} INSTALL_USING_PYTHON=${INSTALLUSINGPYTHON:-false} MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc" AZCLI_ARCHIVE_ARCHITECTURES="amd64 arm64" -AZCLI_ARCHIVE_VERSION_CODENAMES="stretch bookworm buster bullseye bionic focal jammy" +AZCLI_ARCHIVE_VERSION_CODENAMES="stretch bookworm buster bullseye bionic focal jammy noble" if [ "$(id -u)" -ne 0 ]; then echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.' From 1e44a6741d33d65bfaf340b5af6107f0a95441ce Mon Sep 17 00:00:00 2001 From: Gaurav Saini <147703805+gauravsaini04@users.noreply.github.com> Date: Mon, 3 Jun 2024 22:15:24 +0530 Subject: [PATCH 050/247] [Java] - Document additionalVersions functionality (#987) * [Java] - Document additionalVersions functionality * changes requested * changes as requested in pr review * bump patch version * changes requested --- src/java/devcontainer-feature.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/java/devcontainer-feature.json b/src/java/devcontainer-feature.json index 9ed72ad59..c43862610 100644 --- a/src/java/devcontainer-feature.json +++ b/src/java/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "java", - "version": "1.4.1", + "version": "1.5.0", "name": "Java (via SDKMAN!)", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/java", "description": "Installs Java, SDKMAN! (if not installed), and needed dependencies.", @@ -17,6 +17,11 @@ "default": "latest", "description": "Select or enter a Java version to install" }, + "additionalVersions": { + "type": "string", + "default": "", + "description": "Enter additional Java versions, separated by commas." + }, "jdkDistro": { "type": "string", "proposals": [ From 6a9dd0777c5ac0c9a998728491d0e2e0bca11081 Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Mon, 3 Jun 2024 09:52:05 -0700 Subject: [PATCH 051/247] Automated documentation update (#992) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/java/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/java/README.md b/src/java/README.md index 938cd0f89..1a2d91851 100644 --- a/src/java/README.md +++ b/src/java/README.md @@ -16,6 +16,7 @@ Installs Java, SDKMAN! (if not installed), and needed dependencies. | Options Id | Description | Type | Default Value | |-----|-----|-----|-----| | version | Select or enter a Java version to install | string | latest | +| additionalVersions | Enter additional Java versions, separated by commas. | string | - | | jdkDistro | Select or enter a JDK distribution | string | ms | | installGradle | Install Gradle, a build automation tool for multi-language software development | boolean | false | | gradleVersion | Select or enter a Gradle version | string | latest | From 865f69c6a2683603090be0d8c531da1cbf549c9b Mon Sep 17 00:00:00 2001 From: Prathamesh Zarkar <159782310+prathameshzarkar9@users.noreply.github.com> Date: Wed, 12 Jun 2024 01:06:35 +0530 Subject: [PATCH 052/247] #963 specific powershell module version install (#993) * #963 specific powershell module version install * review comments addressed * added test for the version specific module installation and addressed review comments * Update src/powershell/install.sh * Update src/powershell/install.sh * Update src/powershell/install.sh --------- Co-authored-by: Samruddhi Khandale --- src/powershell/devcontainer-feature.json | 4 ++-- src/powershell/install.sh | 16 +++++++++++++--- test/powershell/install_modules_version.sh | 13 +++++++++++++ test/powershell/scenarios.json | 8 ++++++++ 4 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 test/powershell/install_modules_version.sh diff --git a/src/powershell/devcontainer-feature.json b/src/powershell/devcontainer-feature.json index 82ef39a30..d51b5c03f 100644 --- a/src/powershell/devcontainer-feature.json +++ b/src/powershell/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "powershell", - "version": "1.3.5", + "version": "1.4.0", "name": "PowerShell", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/powershell", "description": "Installs PowerShell along with needed dependencies. Useful for base Dockerfiles that often are missing required install dependencies like gpg.", @@ -18,7 +18,7 @@ "modules": { "type": "string", "default": "", - "description": "Optional comma separated list of PowerShell modules to install." + "description": "Optional comma separated list of PowerShell modules to install. If you need to install a specific version of a module, use '==' to specify the version (e.g. 'az.resources==2.5.0')" }, "powershellProfileURL": { "type": "string", diff --git a/src/powershell/install.sh b/src/powershell/install.sh index 533da6f37..43773b3f6 100755 --- a/src/powershell/install.sh +++ b/src/powershell/install.sh @@ -242,14 +242,24 @@ if [ "${use_github}" = "true" ]; then install_using_github fi -# If PowerShell modules are requested, loop through and install +# If PowerShell modules are requested, loop through and install if [ ${#POWERSHELL_MODULES[@]} -gt 0 ]; then echo "Installing PowerShell Modules: ${POWERSHELL_MODULES}" modules=(`echo ${POWERSHELL_MODULES} | tr ',' ' '`) for i in "${modules[@]}" do - echo "Installing ${i}" - pwsh -Command "Install-Module -Name ${i} -AllowClobber -Force -Scope AllUsers" || continue + module_parts=(`echo ${i} | tr '==' ' '`) + module_name="${module_parts[0]}" + args="-Name ${module_name} -AllowClobber -Force -Scope AllUsers" + if [ "${#module_parts[@]}" -eq 2 ]; then + module_version="${module_parts[1]}" + echo "Installing ${module_name} v${module_version}" + args+=" -RequiredVersion ${module_version}" + else + echo "Installing latest version for ${i} module" + fi + + pwsh -Command "Install-Module $args" || continue done fi diff --git a/test/powershell/install_modules_version.sh b/test/powershell/install_modules_version.sh new file mode 100644 index 000000000..ea1a3a98c --- /dev/null +++ b/test/powershell/install_modules_version.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Extension-specific tests +check "az.resources" pwsh -Command "(Get-Module -ListAvailable -Name Az.Resources).Version.ToString()" | grep 2.5.0 +check "az.storage" pwsh -Command "(Get-Module -ListAvailable -Name Az.Storage).Version.ToString()" | grep 4.3.0 + +# Report result +reportResults diff --git a/test/powershell/scenarios.json b/test/powershell/scenarios.json index 6a810b0c9..757a8cc97 100644 --- a/test/powershell/scenarios.json +++ b/test/powershell/scenarios.json @@ -16,5 +16,13 @@ "powershellProfileURL": "https://raw.githubusercontent.com/codspace/powershell-profile/main/Test-Profile.ps1" } } + }, + "install_modules_version": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "powershell": { + "modules": "az.resources==2.5.0, az.storage==4.3.0" + } + } } } From 22ee16e26000d47f6f2ea03a09d68a7487e4603d Mon Sep 17 00:00:00 2001 From: Dev containers Bot <126614555+devcontainers-bot@users.noreply.github.com> Date: Tue, 11 Jun 2024 14:09:20 -0700 Subject: [PATCH 053/247] Automated documentation update (#998) Automated documentation update [skip ci] Co-authored-by: github-actions --- src/powershell/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/powershell/README.md b/src/powershell/README.md index f018778c0..31199a4e1 100644 --- a/src/powershell/README.md +++ b/src/powershell/README.md @@ -16,7 +16,7 @@ Installs PowerShell along with needed dependencies. Useful for base Dockerfiles | Options Id | Description | Type | Default Value | |-----|-----|-----|-----| | version | Select or enter a version of PowerShell. | string | latest | -| modules | Optional comma separated list of PowerShell modules to install. | string | - | +| modules | Optional comma separated list of PowerShell modules to install. If you need to install a specific version of a module, use '==' to specify the version (e.g. 'az.resources==2.5.0') | string | - | | powershellProfileURL | Optional (publicly accessible) URL to download PowerShell profile. | string | - | ## Customizations From 15320f018d0cd72490ba073edb8900968b864ade Mon Sep 17 00:00:00 2001 From: Steven Date: Fri, 14 Jun 2024 02:02:14 +0200 Subject: [PATCH 054/247] dotnet: add ability to install workloads (#997) * dotnet: add ability to install workloads * Bump dotnet feature version * Update NOTES instead of README * Simplify workloads example * Fix temp-dir path oopsie * Improve log message * Fix typo imstalling->installing * Install all workloads at once, fix DOTNET vars not taking effect --- src/dotnet/NOTES.md | 18 +++++++++++------- src/dotnet/devcontainer-feature.json | 7 ++++++- src/dotnet/install.sh | 17 +++++++++++++++++ src/dotnet/scripts/dotnet-helpers.sh | 19 +++++++++++++++---- test/dotnet/dotnet_helpers.sh | 14 +++++++++++--- test/dotnet/install_dotnet_workloads.sh | 24 ++++++++++++++++++++++++ test/dotnet/scenarios.json | 12 +++++++++++- 7 files changed, 95 insertions(+), 16 deletions(-) create mode 100644 test/dotnet/install_dotnet_workloads.sh diff --git a/src/dotnet/NOTES.md b/src/dotnet/NOTES.md index 578aceaf3..584c90cef 100644 --- a/src/dotnet/NOTES.md +++ b/src/dotnet/NOTES.md @@ -2,8 +2,7 @@ Installing only the latest .NET SDK version (the default). -``` json -{ +``` jsonc "features": { "ghcr.io/devcontainers/features/dotnet:2": "latest" // or "" or {} } @@ -12,7 +11,6 @@ Installing only the latest .NET SDK version (the default). Installing an additional SDK version. Multiple versions can be specified as comma-separated values. ``` json -{ "features": { "ghcr.io/devcontainers/features/dotnet:2": { "additionalVersions": "lts" @@ -23,7 +21,6 @@ Installing an additional SDK version. Multiple versions can be specified as comm Installing specific SDK versions. ``` json -{ "features": { "ghcr.io/devcontainers/features/dotnet:2": { "version": "6.0", @@ -35,7 +32,6 @@ Installing specific SDK versions. Installing a specific SDK feature band. ``` json -{ "features": { "ghcr.io/devcontainers/features/dotnet:2": { "version": "6.0.4xx", @@ -46,7 +42,6 @@ Installing a specific SDK feature band. Installing a specific SDK patch version. ``` json -{ "features": { "ghcr.io/devcontainers/features/dotnet:2": { "version": "6.0.412", @@ -57,7 +52,6 @@ Installing a specific SDK patch version. Installing only the .NET Runtime or the ASP.NET Core Runtime. (The SDK includes all runtimes so this configuration is only useful if you need to run .NET apps without building them from source.) ``` json -{ "features": { "ghcr.io/devcontainers/features/dotnet:2": { "version": "none", @@ -67,6 +61,16 @@ Installing only the .NET Runtime or the ASP.NET Core Runtime. (The SDK includes } ``` +Installing .NET workloads. Multiple workloads can be specified as comma-separated values. + +``` json +"features": { + "ghcr.io/devcontainers/features/dotnet:2": { + "workloads": "aspire, wasm-tools" + } +} +``` + ## OS Support This Feature should work on recent versions of Debian/Ubuntu-based distributions with the `apt` package manager installed. diff --git a/src/dotnet/devcontainer-feature.json b/src/dotnet/devcontainer-feature.json index 8b8ffaf2a..fa80799a5 100644 --- a/src/dotnet/devcontainer-feature.json +++ b/src/dotnet/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "dotnet", - "version": "2.0.6", + "version": "2.1.0", "name": "Dotnet CLI", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/dotnet", "description": "This Feature installs the latest .NET SDK, which includes the .NET CLI and the shared runtime. Options are provided to choose a different version or additional versions.", @@ -32,6 +32,11 @@ "type": "string", "default": "", "description": "Enter additional ASP.NET Core runtime versions, separated by commas. Use 'latest' for the latest version, 'lts' for the latest LTS version, 'X.Y' or 'X.Y.Z' for a specific version." + }, + "workloads": { + "type": "string", + "default": "", + "description": "Enter additional .NET SDK workloads, separated by commas. Use 'dotnet workload search' to learn what workloads are available to install." } }, "containerEnv": { diff --git a/src/dotnet/install.sh b/src/dotnet/install.sh index 237a8a0be..d289ea2a4 100644 --- a/src/dotnet/install.sh +++ b/src/dotnet/install.sh @@ -10,6 +10,14 @@ DOTNET_VERSION="${VERSION:-"latest"}" ADDITIONAL_VERSIONS="${ADDITIONALVERSIONS:-""}" DOTNET_RUNTIME_VERSIONS="${DOTNETRUNTIMEVERSIONS:-""}" ASPNETCORE_RUNTIME_VERSIONS="${ASPNETCORERUNTIMEVERSIONS:-""}" +WORKLOADS="${WORKLOADS:-""}" + +# Prevent "Welcome to .NET" message from dotnet +export DOTNET_NOLOGO=true + +# Prevent generating a development certificate while running this script +# Otherwise it would be stored in the image, which is undesirable +export DOTNET_GENERATE_ASPNET_CERTIFICATE=false set -e @@ -111,6 +119,15 @@ for version in "${aspNetCoreRuntimeVersions[@]}"; do install_runtime "aspnetcore" "$version" done +workloads=() +for workload in $(split_csv "$WORKLOADS"); do + workloads+=("$workload") +done + +if [ ${#workloads[@]} -ne 0 ]; then + install_workloads "${workloads[@]}" +fi + # Clean up rm -rf /var/lib/apt/lists/* rm -rf scripts diff --git a/src/dotnet/scripts/dotnet-helpers.sh b/src/dotnet/scripts/dotnet-helpers.sh index bda0c9c3a..b7024c9d1 100644 --- a/src/dotnet/scripts/dotnet-helpers.sh +++ b/src/dotnet/scripts/dotnet-helpers.sh @@ -25,7 +25,6 @@ fetch_latest_version_in_channel() { else wget -qO- "https://dotnetcli.azureedge.net/dotnet/Sdk/$channel/latest.version" fi - } # Prints the latest dotnet version @@ -76,12 +75,11 @@ install_sdk() { fi # Currently this script does not make it possible to qualify the version, 'GA' is always implied - echo "Executing $DOTNET_INSTALL_SCRIPT --version $version --channel $channel --install-dir $DOTNET_INSTALL_DIR --no-path" + echo "Executing $DOTNET_INSTALL_SCRIPT --version $version --channel $channel --install-dir $DOTNET_INSTALL_DIR" "$DOTNET_INSTALL_SCRIPT" \ --version "$version" \ --channel "$channel" \ - --install-dir "$DOTNET_INSTALL_DIR" \ - --no-path + --install-dir "$DOTNET_INSTALL_DIR" } # Installs a version of the .NET Runtime @@ -117,3 +115,16 @@ install_runtime() { --install-dir "$DOTNET_INSTALL_DIR" \ --no-path } + +# Installs one or more .NET workloads +# Usage: install_workload [ ...] +# Reference: https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-workload-install +install_workloads() { + local workloads="$@" + + echo "Installing .NET workload(s) $workloads" + dotnet workload install $workloads --temp-dir /tmp/dotnet-workload-temp-dir + + # Clean up + rm -r /tmp/dotnet-workload-temp-dir +} diff --git a/test/dotnet/dotnet_helpers.sh b/test/dotnet/dotnet_helpers.sh index 6c833b444..a24bd1ce2 100644 --- a/test/dotnet/dotnet_helpers.sh +++ b/test/dotnet/dotnet_helpers.sh @@ -15,7 +15,6 @@ fetch_latest_version_in_channel() { else wget -qO- "https://dotnetcli.azureedge.net/dotnet/Sdk/$channel/latest.version" fi - } # Prints the latest dotnet version @@ -47,7 +46,6 @@ is_dotnet_sdk_version_installed() { return $? } - # Asserts that the specified .NET Runtime version is installed # Returns a non-zero exit code if the check fails # Usage: is_dotnet_runtime_version_installed @@ -68,4 +66,14 @@ is_aspnetcore_runtime_version_installed() { local expected="$1" dotnet --list-runtimes | grep --fixed-strings --silent "Microsoft.AspNetCore.App $expected" return $? -} \ No newline at end of file +} + +# Asserts that the specified workload is installed +# Returns a non-zero exit code if the check fails +# Usage: is_dotnet_workload_installed +# Example: is_dotnet_workload_installed "aspire" +is_dotnet_workload_installed() { + local expected="$1" + dotnet workload list | grep --fixed-strings --silent "$expected" + return $? +} diff --git a/test/dotnet/install_dotnet_workloads.sh b/test/dotnet/install_dotnet_workloads.sh new file mode 100644 index 000000000..37c86a2d4 --- /dev/null +++ b/test/dotnet/install_dotnet_workloads.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +# Optional: Import test library bundled with the devcontainer CLI +# See https://github.com/devcontainers/cli/blob/HEAD/docs/features/test.md#dev-container-features-test-lib +# Provides the 'check' and 'reportResults' commands. +source dev-container-features-test-lib + +# Feature-specific tests +# The 'check' command comes from the dev-container-features-test-lib. Syntax is... +# check