From 2f67147bbbda59ff2feda116130df97d843a0229 Mon Sep 17 00:00:00 2001 From: gauravsaini04 <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:29:59 +0000 Subject: [PATCH 01/21] implementation of fallback logic for cosign, terraform --- src/terraform/install.sh | 58 +++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/src/terraform/install.sh b/src/terraform/install.sh index c1a382597..cee0a0410 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -177,6 +177,37 @@ check_packages() { fi } +# Function to fetch the version released prior to the latest version +get_previous_version() { + REPO_URL=$1 + curl -s "${REPO_URL}/latest" | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/' +} + +install_prev_vers() { + PKG_NAME=$1 + FAILED_VERSION=$2 + REPO_URL=$3 + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${FAILED_VERSION}..." + PREVIOUS_VERSION=$(get_previous_version "${REPO_URL}") + echo -e "\nAttempting to install ${PREVIOUS_VERSION}" + echo "The installed version: ${PREVIOUS_VERSION#v}" + INSTALLER_FN="install_${PKG_NAME}" + $INSTALLER_FN "${PREVIOUS_VERSION#v}" +} + +install_cosign() { + COSIGN_VERSION=$1 + curl -L "https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" -o /tmp/cosign_${COSIGN_VERSION}_${architecture}.deb + output=$(cat /tmp/cosign_${COSIGN_VERSION}_${architecture}.deb) + if [[ "$output" != *"Not Found"* ]]; then + dpkg -i /tmp/cosign_${COSIGN_VERSION}_${architecture}.deb + rm /tmp/cosign_${COSIGN_VERSION}_${architecture}.deb + echo "Installation of cosign succeeded with ${COSIGN_VERSION}." + else + echo "Installation Failed." + fi +} + # Install 'cosign' for validating signatures # https://docs.sigstore.dev/cosign/overview/ ensure_cosign() { @@ -186,10 +217,12 @@ ensure_cosign() { echo "Installing cosign..." LATEST_COSIGN_VERSION="latest" find_version_from_git_tags LATEST_COSIGN_VERSION 'https://github.com/sigstore/cosign' - curl -L "https://github.com/sigstore/cosign/releases/latest/download/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb" -o /tmp/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb - - dpkg -i /tmp/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb - rm /tmp/cosign_${LATEST_COSIGN_VERSION}_${architecture}.deb + # LATEST_COSIGN_VERSION="1.2.xyz" + INSTALL_STATUS=$(install_cosign "${LATEST_COSIGN_VERSION}"); + if [ "${INSTALL_STATUS}" == "Installation Failed." ]; then + LATEST_COSIGN_VERSION=$(install_prev_vers "cosign" "${LATEST_COSIGN_VERSION}" "https://api.github.com/repos/sigstore/cosign/releases" | grep "The installed version"); + LATEST_COSIGN_VERSION=$(echo "$LATEST_COSIGN_VERSION" | sed 's/The installed version: //') + fi fi if ! type cosign > /dev/null 2>&1; then echo "(!) Failed to install cosign." @@ -212,13 +245,24 @@ find_version_from_git_tags TERRAFORM_VERSION 'https://github.com/hashicorp/terra find_version_from_git_tags TFLINT_VERSION 'https://github.com/terraform-linters/tflint' find_version_from_git_tags TERRAGRUNT_VERSION 'https://github.com/gruntwork-io/terragrunt' +install_terraform() { + TERRAFORM_VERSION=$1 + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + curl -sSL -o ${terraform_filename} "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${terraform_filename}" + echo "${terraform_filename}" +} + +# TERRAFORM_VERSION="1.2.xyz" mkdir -p /tmp/tf-downloads cd /tmp/tf-downloads - # Install Terraform, tflint, Terragrunt echo "Downloading terraform..." -terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" -curl -sSL -o ${terraform_filename} "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${terraform_filename}" +terraform_filename=$(install_terraform "${TERRAFORM_VERSION}"); +if grep -q "The specified key does not exist." "${terraform_filename}"; then + TERRAFORM_VERSION=$(install_prev_vers "terraform" "${TERRAFORM_VERSION}" "https://api.github.com/repos/hashicorp/terraform/releases" | grep "The installed version"); + TERRAFORM_VERSION=$( echo "$TERRAFORM_VERSION" | sed 's/The installed version: //'); + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" +fi if [ "${TERRAFORM_SHA256}" != "dev-mode" ]; then if [ "${TERRAFORM_SHA256}" = "automatic" ]; then receive_gpg_keys TERRAFORM_GPG_KEY From d7bd148cda46e657732bacace788b81689d3b0ed Mon Sep 17 00:00:00 2001 From: gauravsaini04 <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:38:33 +0000 Subject: [PATCH 02/21] tflint, terragrunt - wrote fallback logic for these --- src/terraform/install.sh | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/terraform/install.sh b/src/terraform/install.sh index cee0a0410..7a8402919 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -277,10 +277,21 @@ fi unzip ${terraform_filename} mv -f terraform /usr/local/bin/ -if [ "${TFLINT_VERSION}" != "none" ]; then - echo "Downloading tflint..." +install_tflint() { + TFLINT_VERSION=$1 TFLINT_FILENAME="tflint_linux_${architecture}.zip" curl -sSL -o /tmp/tf-downloads/${TFLINT_FILENAME} https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/${TFLINT_FILENAME} + echo "${TFLINT_FILENAME}" +} + +# TFLINT_VERSION="1.2.xyz" +if [ "${TFLINT_VERSION}" != "none" ]; then + echo "Downloading tflint..." + TFLINT_FILENAME=$(install_tflint "${TFLINT_VERSION}"); + if grep -q "Not Found" "/tmp/tf-downloads/${TFLINT_FILENAME}"; then + TFLINT_VERSION=$(install_prev_vers "tflint" "${TFLINT_VERSION}" "https://api.github.com/repos/terraform-linters/tflint/releases" | grep "The installed version") + TFLINT_VERSION=$(echo "${TFLINT_VERSION}" | sed 's/The installed version: //'); + fi if [ "${TFLINT_SHA256}" != "dev-mode" ]; then if [ "${TFLINT_SHA256}" != "automatic" ]; then @@ -321,10 +332,22 @@ if [ "${TFLINT_VERSION}" != "none" ]; then unzip /tmp/tf-downloads/${TFLINT_FILENAME} mv -f tflint /usr/local/bin/ fi -if [ "${TERRAGRUNT_VERSION}" != "none" ]; then - echo "Downloading Terragrunt..." + +install_terragrunt() { + TERRAGRUNT_VERSION=$1 terragrunt_filename="terragrunt_linux_${architecture}" curl -sSL -o /tmp/tf-downloads/${terragrunt_filename} https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/${terragrunt_filename} + echo "${terragrunt_filename}" +} + +TERRAGRUNT_VERSION="1.2.xyz" +if [ "${TERRAGRUNT_VERSION}" != "none" ]; then + echo "Downloading Terragrunt..." + terragrunt_filename=$(install_terragrunt "${TERRAGRUNT_VERSION}") + if grep -q "Not Found" "/tmp/tf-downloads/${terragrunt_filename}"; then + TERRAGRUNT_VERSION=$(install_prev_vers "terragrunt" "${TERRAGRUNT_VERSION}" "https://api.github.com/repos/gruntwork-io/terragrunt/releases" | grep "The installed version"); + TERRAGRUNT_VERSION=$(echo "${TERRAGRUNT_VERSION}" | sed 's/The installed version: //'); + fi if [ "${TERRAGRUNT_SHA256}" != "dev-mode" ]; then if [ "${TERRAGRUNT_SHA256}" = "automatic" ]; then curl -sSL -o terragrunt_SHA256SUMS https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/SHA256SUMS From 39ab5bb44b5bca8b51d7fbf7dcb34c63a7c980b3 Mon Sep 17 00:00:00 2001 From: gauravsaini04 <147703805+gauravsaini04@users.noreply.github.com> Date: Wed, 13 Mar 2024 23:23:14 +0000 Subject: [PATCH 03/21] misc change --- src/terraform/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/terraform/install.sh b/src/terraform/install.sh index 7a8402919..a0200dc57 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -340,7 +340,7 @@ install_terragrunt() { echo "${terragrunt_filename}" } -TERRAGRUNT_VERSION="1.2.xyz" +# TERRAGRUNT_VERSION="1.2.xyz" if [ "${TERRAGRUNT_VERSION}" != "none" ]; then echo "Downloading Terragrunt..." terragrunt_filename=$(install_terragrunt "${TERRAGRUNT_VERSION}") From a2c8da6292a2cda76d5f7e8546d94e26811fa1c2 Mon Sep 17 00:00:00 2001 From: gauravsaini04 <147703805+gauravsaini04@users.noreply.github.com> Date: Thu, 14 Mar 2024 15:35:09 +0000 Subject: [PATCH 04/21] [Terraform] - fallback - apply --- src/terraform/install.sh | 48 +++++++--- test/terraform/scenarios.json | 16 ++++ .../terraform/terraform_docs_fallback_test.sh | 92 +++++++++++++++++++ test/terraform/tfsec_fallback_test.sh | 89 ++++++++++++++++++ 4 files changed, 230 insertions(+), 15 deletions(-) create mode 100644 test/terraform/terraform_docs_fallback_test.sh create mode 100644 test/terraform/tfsec_fallback_test.sh diff --git a/src/terraform/install.sh b/src/terraform/install.sh index a0200dc57..bd5d83169 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -217,7 +217,6 @@ ensure_cosign() { echo "Installing cosign..." LATEST_COSIGN_VERSION="latest" find_version_from_git_tags LATEST_COSIGN_VERSION 'https://github.com/sigstore/cosign' - # LATEST_COSIGN_VERSION="1.2.xyz" INSTALL_STATUS=$(install_cosign "${LATEST_COSIGN_VERSION}"); if [ "${INSTALL_STATUS}" == "Installation Failed." ]; then LATEST_COSIGN_VERSION=$(install_prev_vers "cosign" "${LATEST_COSIGN_VERSION}" "https://api.github.com/repos/sigstore/cosign/releases" | grep "The installed version"); @@ -249,18 +248,17 @@ install_terraform() { TERRAFORM_VERSION=$1 terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" curl -sSL -o ${terraform_filename} "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${terraform_filename}" - echo "${terraform_filename}" } -# TERRAFORM_VERSION="1.2.xyz" mkdir -p /tmp/tf-downloads cd /tmp/tf-downloads # Install Terraform, tflint, Terragrunt echo "Downloading terraform..." -terraform_filename=$(install_terraform "${TERRAFORM_VERSION}"); +terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" +install_terraform "$TERRAFORM_VERSION" if grep -q "The specified key does not exist." "${terraform_filename}"; then TERRAFORM_VERSION=$(install_prev_vers "terraform" "${TERRAFORM_VERSION}" "https://api.github.com/repos/hashicorp/terraform/releases" | grep "The installed version"); - TERRAFORM_VERSION=$( echo "$TERRAFORM_VERSION" | sed 's/The installed version: //'); + TERRAFORM_VERSION=$(echo "$TERRAFORM_VERSION" | sed 's/The installed version: //'); terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" fi if [ "${TERRAFORM_SHA256}" != "dev-mode" ]; then @@ -278,16 +276,15 @@ unzip ${terraform_filename} mv -f terraform /usr/local/bin/ install_tflint() { - TFLINT_VERSION=$1 + local TFLINT_VERSION=$1 TFLINT_FILENAME="tflint_linux_${architecture}.zip" curl -sSL -o /tmp/tf-downloads/${TFLINT_FILENAME} https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/${TFLINT_FILENAME} - echo "${TFLINT_FILENAME}" } -# TFLINT_VERSION="1.2.xyz" if [ "${TFLINT_VERSION}" != "none" ]; then echo "Downloading tflint..." - TFLINT_FILENAME=$(install_tflint "${TFLINT_VERSION}"); + TFLINT_FILENAME="tflint_linux_${architecture}.zip" + install_tflint "$TFLINT_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${TFLINT_FILENAME}"; then TFLINT_VERSION=$(install_prev_vers "tflint" "${TFLINT_VERSION}" "https://api.github.com/repos/terraform-linters/tflint/releases" | grep "The installed version") TFLINT_VERSION=$(echo "${TFLINT_VERSION}" | sed 's/The installed version: //'); @@ -334,16 +331,15 @@ if [ "${TFLINT_VERSION}" != "none" ]; then fi install_terragrunt() { - TERRAGRUNT_VERSION=$1 + local TERRAGRUNT_VERSION=$1 terragrunt_filename="terragrunt_linux_${architecture}" curl -sSL -o /tmp/tf-downloads/${terragrunt_filename} https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/${terragrunt_filename} - echo "${terragrunt_filename}" } -# TERRAGRUNT_VERSION="1.2.xyz" if [ "${TERRAGRUNT_VERSION}" != "none" ]; then echo "Downloading Terragrunt..." - terragrunt_filename=$(install_terragrunt "${TERRAGRUNT_VERSION}") + terragrunt_filename="terragrunt_linux_${architecture}" + install_terragrunt "$TERRAGRUNT_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${terragrunt_filename}"; then TERRAGRUNT_VERSION=$(install_prev_vers "terragrunt" "${TERRAGRUNT_VERSION}" "https://api.github.com/repos/gruntwork-io/terragrunt/releases" | grep "The installed version"); TERRAGRUNT_VERSION=$(echo "${TERRAGRUNT_VERSION}" | sed 's/The installed version: //'); @@ -385,12 +381,23 @@ if [ "${INSTALL_SENTINEL}" = "true" ]; then mv -f /tmp/tf-downloads/sentinel /usr/local/bin/sentinel fi +install_tfsec() { + TFSEC_VERSION=$1 + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfsec_filename} https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${tfsec_filename} +} + if [ "${INSTALL_TFSEC}" = "true" ]; then TFSEC_VERSION="latest" find_version_from_git_tags TFSEC_VERSION 'https://github.com/aquasecurity/tfsec' tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" echo "(*) Downloading TFSec... ${tfsec_filename}" - curl -sSL -o /tmp/tf-downloads/${tfsec_filename} https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${tfsec_filename} + install_tfsec "$TFSEC_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfsec_filename}"; then + TFSEC_VERSION=$(install_prev_vers "tfsec" "${TFSEC_VERSION}" "https://api.github.com/repos/aquasecurity/tfsec/releases" | grep "The installed version"); + TFSEC_VERSION=$(echo "${TFSEC_VERSION}" | sed 's/The installed version: //'); + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + fi if [ "${TFSEC_SHA256}" != "dev-mode" ]; then if [ "${TFSEC_SHA256}" = "automatic" ]; then curl -sSL -o tfsec_SHA256SUMS https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/tfsec_${TFSEC_VERSION}_checksums.txt @@ -405,12 +412,23 @@ if [ "${INSTALL_TFSEC}" = "true" ]; then mv -f /tmp/tf-downloads/tfsec/tfsec /usr/local/bin/tfsec fi +install_tfdocs() { + TERRAFORM_DOCS_VERSION=$1 + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfdocs_filename} https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/${tfdocs_filename} +} + if [ "${INSTALL_TERRAFORM_DOCS}" = "true" ]; then TERRAFORM_DOCS_VERSION="latest" find_version_from_git_tags TERRAFORM_DOCS_VERSION 'https://github.com/terraform-docs/terraform-docs' tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" echo "(*) Downloading Terraform docs... ${tfdocs_filename}" - curl -sSL -o /tmp/tf-downloads/${tfdocs_filename} https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/${tfdocs_filename} + install_tfdocs "$TERRAFORM_DOCS_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfdocs_filename}"; then + TERRAFORM_DOCS_VERSION=$(install_prev_vers "tfdocs" "${TERRAFORM_DOCS_VERSION}" "https://api.github.com/repos/terraform-docs/terraform-docs/releases" | grep "The installed version"); + TERRAFORM_DOCS_VERSION=$(echo "${TERRAFORM_DOCS_VERSION}" | sed 's/The installed version: //'); + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + fi if [ "${TERRAFORM_DOCS_SHA256}" != "dev-mode" ]; then if [ "${TERRAFORM_DOCS_SHA256}" = "automatic" ]; then curl -sSL -o tfdocs_SHA256SUMS https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/terraform-docs-v${TERRAFORM_DOCS_VERSION}.sha256sum diff --git a/test/terraform/scenarios.json b/test/terraform/scenarios.json index 0a4d8956c..299e97cff 100644 --- a/test/terraform/scenarios.json +++ b/test/terraform/scenarios.json @@ -15,6 +15,14 @@ } } }, + "tfsec_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "installTFsec": true + } + } + }, "install_terraform_docs": { "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { @@ -23,6 +31,14 @@ } } }, + "terraform_docs_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "installTerraformDocs": true + } + } + }, "older_tflint": { "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { diff --git a/test/terraform/terraform_docs_fallback_test.sh b/test/terraform/terraform_docs_fallback_test.sh new file mode 100644 index 000000000..6d1bda193 --- /dev/null +++ b/test/terraform/terraform_docs_fallback_test.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +# Terraform Docs specific tests +check "terraform-docs version as installed by feature" terraform-docs --version + +TERRAFORM_DOCS_SHA256="automatic" + +handle_error() { + echo "Error occurred on line ${BASH_LINENO[0]}" + exit 1 +} + +# Trap errors and call the error handling function +trap 'handle_error' ERR + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Function to fetch the version released prior to the latest version +get_previous_version() { + REPO_URL=$1 + curl -s "${REPO_URL}/latest" | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/' +} + +install_prev_vers() { + PKG_NAME=$1 + FAILED_VERSION=$2 + REPO_URL=$3 + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${FAILED_VERSION}..." + PREVIOUS_VERSION=$(get_previous_version "${REPO_URL}") + echo -e "\nAttempting to install ${PREVIOUS_VERSION}" + echo "The installed version: ${PREVIOUS_VERSION#v}" + INSTALLER_FN="install_${PKG_NAME}" + $INSTALLER_FN "${PREVIOUS_VERSION#v}" +} + +install_tfdocs() { + TERRAFORM_DOCS_VERSION=$1 + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfdocs_filename} https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/${tfdocs_filename} +} + +try_install_terraform_docs_dummy_version() { + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + TERRAFORM_DOCS_VERSION="1.2.xyz" + echo -e "\nInstalling TERRAFORM_DOCS dummy version.." v${TERRAFORM_DOCS_VERSION} + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + echo "(*) Downloading Terraform docs... ${tfdocs_filename}" + install_tfdocs "$TERRAFORM_DOCS_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfdocs_filename}"; then + TERRAFORM_DOCS_VERSION=$(install_prev_vers "tfdocs" "${TERRAFORM_DOCS_VERSION}" "https://api.github.com/repos/terraform-docs/terraform-docs/releases" | grep "The installed version"); + TERRAFORM_DOCS_VERSION=$(echo "${TERRAFORM_DOCS_VERSION}" | sed 's/The installed version: //'); + tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" + fi + if [ "${TERRAFORM_DOCS_SHA256}" != "dev-mode" ]; then + if [ "${TERRAFORM_DOCS_SHA256}" = "automatic" ]; then + curl -sSL -o tfdocs_SHA256SUMS https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/terraform-docs-v${TERRAFORM_DOCS_VERSION}.sha256sum + else + echo "${TERRAFORM_DOCS_SHA256} *${tfsec_filename}" > tfdocs_SHA256SUMS + fi + sha256sum --ignore-missing -c tfdocs_SHA256SUMS + fi + mkdir -p /tmp/tf-downloads/tfdocs + tar -xzf /tmp/tf-downloads/${tfdocs_filename} -C /tmp/tf-downloads/tfdocs + sudo chmod a+x /tmp/tf-downloads/tfdocs/terraform-docs + sudo mv -f /tmp/tf-downloads/tfdocs/terraform-docs /usr/local/bin/terraform-docs + + rm -rf /tmp/tf-downloads ${GNUPGHOME} +} + +try_install_terraform_docs_dummy_version + + +check "terraform-docs version as installed by test" terraform-docs --version + +# Report result +reportResults \ No newline at end of file diff --git a/test/terraform/tfsec_fallback_test.sh b/test/terraform/tfsec_fallback_test.sh new file mode 100644 index 000000000..a17d00e25 --- /dev/null +++ b/test/terraform/tfsec_fallback_test.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +handle_error() { + echo "Error occurred on line ${BASH_LINENO[0]}" + exit 1 +} + +TFSEC_SHA256="automatic" + +# Trap errors and call the error handling function +trap 'handle_error' ERR + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# TFSec specific tests +check "tfsec version as installed by feature" tfsec --version + +# Function to fetch the version released prior to the latest version +get_previous_version() { + REPO_URL=$1 + curl -s "${REPO_URL}/latest" | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/' +} + +install_prev_vers() { + PKG_NAME=$1 + FAILED_VERSION=$2 + REPO_URL=$3 + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${FAILED_VERSION}..." + PREVIOUS_VERSION=$(get_previous_version "${REPO_URL}") + echo -e "\nAttempting to install ${PREVIOUS_VERSION}" + echo "The installed version: ${PREVIOUS_VERSION#v}" + INSTALLER_FN="install_${PKG_NAME}" + $INSTALLER_FN "${PREVIOUS_VERSION#v}" +} + +install_tfsec() { + TFSEC_VERSION=$1 + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + curl -sSL -o /tmp/tf-downloads/${tfsec_filename} https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${tfsec_filename} +} + +try_install_tfsec_dummy_version() { + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + TFSEC_VERSION="1.2.xyz" + echo -e "\nInstalling TFSEC dummy version.." v${TFSEC_VERSION} + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + echo "(*) Downloading TFSec... ${tfsec_filename}" + install_tfsec "$TFSEC_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${tfsec_filename}"; then + TFSEC_VERSION=$(install_prev_vers "tfsec" "${TFSEC_VERSION}" "https://api.github.com/repos/aquasecurity/tfsec/releases" | grep "The installed version"); + TFSEC_VERSION=$(echo "${TFSEC_VERSION}" | sed 's/The installed version: //'); + tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" + fi + if [ "${TFSEC_SHA256}" != "dev-mode" ]; then + if [ "${TFSEC_SHA256}" = "automatic" ]; then + curl -sSL -o tfsec_SHA256SUMS https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/tfsec_${TFSEC_VERSION}_checksums.txt + else + echo "${TFSEC_SHA256} *${tfsec_filename}" > tfsec_SHA256SUMS + fi + sha256sum --ignore-missing -c tfsec_SHA256SUMS + fi + mkdir -p /tmp/tf-downloads/tfsec + tar -xzf /tmp/tf-downloads/${tfsec_filename} -C /tmp/tf-downloads/tfsec + chmod a+x /tmp/tf-downloads/tfsec/tfsec + sudo mv -f /tmp/tf-downloads/tfsec/tfsec /usr/local/bin/tfsec +} + +try_install_tfsec_dummy_version + +check "tfsec version as installed by test after fallbacking from the dummy version" tfsec --version + +# Report result +reportResults \ No newline at end of file From e8b8be3162c5a7a65231ba9aea73bf9786192461 Mon Sep 17 00:00:00 2001 From: gauravsaini04 <147703805+gauravsaini04@users.noreply.github.com> Date: Sun, 17 Mar 2024 14:23:14 +0000 Subject: [PATCH 05/21] changes as requested by review comments --- src/terraform/devcontainer-feature.json | 2 +- src/terraform/install.sh | 54 +++++++++---------- .../terraform/terraform_docs_fallback_test.sh | 25 ++++----- test/terraform/tfsec_fallback_test.sh | 25 ++++----- 4 files changed, 52 insertions(+), 54 deletions(-) diff --git a/src/terraform/devcontainer-feature.json b/src/terraform/devcontainer-feature.json index 51fb9a27a..5c9c042cc 100644 --- a/src/terraform/devcontainer-feature.json +++ b/src/terraform/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "terraform", - "version": "1.3.5", + "version": "1.3.6", "name": "Terraform, tflint, and TFGrunt", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/terraform", "description": "Installs the Terraform CLI and optionally TFLint and Terragrunt. Auto-detects latest version and installs needed dependencies.", diff --git a/src/terraform/install.sh b/src/terraform/install.sh index bd5d83169..734e302a5 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -180,23 +180,25 @@ check_packages() { # Function to fetch the version released prior to the latest version get_previous_version() { REPO_URL=$1 - curl -s "${REPO_URL}/latest" | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/' + curl -s "${REPO_URL}/latest" | jq -r '.tag_name' } -install_prev_vers() { - PKG_NAME=$1 - FAILED_VERSION=$2 - REPO_URL=$3 - echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${FAILED_VERSION}..." - PREVIOUS_VERSION=$(get_previous_version "${REPO_URL}") - echo -e "\nAttempting to install ${PREVIOUS_VERSION}" - echo "The installed version: ${PREVIOUS_VERSION#v}" +install_previous_version() { + local given_version=$1 + local requested_version=${!given_version} + local PKG_NAME=$2 + local REPO_URL=$3 + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + requested_version=$(get_previous_version "${REPO_URL}") + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" INSTALLER_FN="install_${PKG_NAME}" - $INSTALLER_FN "${PREVIOUS_VERSION#v}" + $INSTALLER_FN "${requested_version#v}" + echo "${given_version}=${!given_version}" } install_cosign() { - COSIGN_VERSION=$1 + local COSIGN_VERSION=$1 curl -L "https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" -o /tmp/cosign_${COSIGN_VERSION}_${architecture}.deb output=$(cat /tmp/cosign_${COSIGN_VERSION}_${architecture}.deb) if [[ "$output" != *"Not Found"* ]]; then @@ -218,9 +220,8 @@ ensure_cosign() { LATEST_COSIGN_VERSION="latest" find_version_from_git_tags LATEST_COSIGN_VERSION 'https://github.com/sigstore/cosign' INSTALL_STATUS=$(install_cosign "${LATEST_COSIGN_VERSION}"); - if [ "${INSTALL_STATUS}" == "Installation Failed." ]; then - LATEST_COSIGN_VERSION=$(install_prev_vers "cosign" "${LATEST_COSIGN_VERSION}" "https://api.github.com/repos/sigstore/cosign/releases" | grep "The installed version"); - LATEST_COSIGN_VERSION=$(echo "$LATEST_COSIGN_VERSION" | sed 's/The installed version: //') + if [[ "${INSTALL_STATUS}"=="Installation Failed." ]]; then + install_previous_version LATEST_COSIGN_VERSION "cosign" "https://api.github.com/repos/sigstore/cosign/releases" fi fi if ! type cosign > /dev/null 2>&1; then @@ -245,7 +246,7 @@ find_version_from_git_tags TFLINT_VERSION 'https://github.com/terraform-linters/ find_version_from_git_tags TERRAGRUNT_VERSION 'https://github.com/gruntwork-io/terragrunt' install_terraform() { - TERRAFORM_VERSION=$1 + local TERRAFORM_VERSION=$1 terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" curl -sSL -o ${terraform_filename} "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${terraform_filename}" } @@ -257,8 +258,7 @@ echo "Downloading terraform..." terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" install_terraform "$TERRAFORM_VERSION" if grep -q "The specified key does not exist." "${terraform_filename}"; then - TERRAFORM_VERSION=$(install_prev_vers "terraform" "${TERRAFORM_VERSION}" "https://api.github.com/repos/hashicorp/terraform/releases" | grep "The installed version"); - TERRAFORM_VERSION=$(echo "$TERRAFORM_VERSION" | sed 's/The installed version: //'); + install_previous_version TERRAFORM_VERSION "terraform" "https://api.github.com/repos/hashicorp/terraform/releases" terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" fi if [ "${TERRAFORM_SHA256}" != "dev-mode" ]; then @@ -276,7 +276,7 @@ unzip ${terraform_filename} mv -f terraform /usr/local/bin/ install_tflint() { - local TFLINT_VERSION=$1 + TFLINT_VERSION=$1 TFLINT_FILENAME="tflint_linux_${architecture}.zip" curl -sSL -o /tmp/tf-downloads/${TFLINT_FILENAME} https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/${TFLINT_FILENAME} } @@ -286,8 +286,7 @@ if [ "${TFLINT_VERSION}" != "none" ]; then TFLINT_FILENAME="tflint_linux_${architecture}.zip" install_tflint "$TFLINT_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${TFLINT_FILENAME}"; then - TFLINT_VERSION=$(install_prev_vers "tflint" "${TFLINT_VERSION}" "https://api.github.com/repos/terraform-linters/tflint/releases" | grep "The installed version") - TFLINT_VERSION=$(echo "${TFLINT_VERSION}" | sed 's/The installed version: //'); + install_previous_version TFLINT_VERSION "tflint" "https://api.github.com/repos/terraform-linters/tflint/releases" fi if [ "${TFLINT_SHA256}" != "dev-mode" ]; then @@ -331,7 +330,7 @@ if [ "${TFLINT_VERSION}" != "none" ]; then fi install_terragrunt() { - local TERRAGRUNT_VERSION=$1 + TERRAGRUNT_VERSION=$1 terragrunt_filename="terragrunt_linux_${architecture}" curl -sSL -o /tmp/tf-downloads/${terragrunt_filename} https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/${terragrunt_filename} } @@ -341,8 +340,7 @@ if [ "${TERRAGRUNT_VERSION}" != "none" ]; then terragrunt_filename="terragrunt_linux_${architecture}" install_terragrunt "$TERRAGRUNT_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${terragrunt_filename}"; then - TERRAGRUNT_VERSION=$(install_prev_vers "terragrunt" "${TERRAGRUNT_VERSION}" "https://api.github.com/repos/gruntwork-io/terragrunt/releases" | grep "The installed version"); - TERRAGRUNT_VERSION=$(echo "${TERRAGRUNT_VERSION}" | sed 's/The installed version: //'); + install_previous_version TERRAGRUNT_VERSION "terragrunt" "https://api.github.com/repos/gruntwork-io/terragrunt/releases" fi if [ "${TERRAGRUNT_SHA256}" != "dev-mode" ]; then if [ "${TERRAGRUNT_SHA256}" = "automatic" ]; then @@ -382,7 +380,7 @@ if [ "${INSTALL_SENTINEL}" = "true" ]; then fi install_tfsec() { - TFSEC_VERSION=$1 + local TFSEC_VERSION=$1 tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" curl -sSL -o /tmp/tf-downloads/${tfsec_filename} https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${tfsec_filename} } @@ -394,8 +392,7 @@ if [ "${INSTALL_TFSEC}" = "true" ]; then echo "(*) Downloading TFSec... ${tfsec_filename}" install_tfsec "$TFSEC_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${tfsec_filename}"; then - TFSEC_VERSION=$(install_prev_vers "tfsec" "${TFSEC_VERSION}" "https://api.github.com/repos/aquasecurity/tfsec/releases" | grep "The installed version"); - TFSEC_VERSION=$(echo "${TFSEC_VERSION}" | sed 's/The installed version: //'); + install_previous_version TFSEC_VERSION "tfsec" "https://api.github.com/repos/aquasecurity/tfsec/releases" tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" fi if [ "${TFSEC_SHA256}" != "dev-mode" ]; then @@ -413,7 +410,7 @@ if [ "${INSTALL_TFSEC}" = "true" ]; then fi install_tfdocs() { - TERRAFORM_DOCS_VERSION=$1 + local TERRAFORM_DOCS_VERSION=$1 tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" curl -sSL -o /tmp/tf-downloads/${tfdocs_filename} https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/${tfdocs_filename} } @@ -425,8 +422,7 @@ if [ "${INSTALL_TERRAFORM_DOCS}" = "true" ]; then echo "(*) Downloading Terraform docs... ${tfdocs_filename}" install_tfdocs "$TERRAFORM_DOCS_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${tfdocs_filename}"; then - TERRAFORM_DOCS_VERSION=$(install_prev_vers "tfdocs" "${TERRAFORM_DOCS_VERSION}" "https://api.github.com/repos/terraform-docs/terraform-docs/releases" | grep "The installed version"); - TERRAFORM_DOCS_VERSION=$(echo "${TERRAFORM_DOCS_VERSION}" | sed 's/The installed version: //'); + install_previous_version TERRAFORM_DOCS_VERSION "tfdocs" "https://api.github.com/repos/terraform-docs/terraform-docs/releases" tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" fi if [ "${TERRAFORM_DOCS_SHA256}" != "dev-mode" ]; then diff --git a/test/terraform/terraform_docs_fallback_test.sh b/test/terraform/terraform_docs_fallback_test.sh index 6d1bda193..4a4964335 100644 --- a/test/terraform/terraform_docs_fallback_test.sh +++ b/test/terraform/terraform_docs_fallback_test.sh @@ -33,19 +33,21 @@ esac # Function to fetch the version released prior to the latest version get_previous_version() { REPO_URL=$1 - curl -s "${REPO_URL}/latest" | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/' + curl -s "${REPO_URL}/latest" | jq -r '.tag_name' } -install_prev_vers() { - PKG_NAME=$1 - FAILED_VERSION=$2 - REPO_URL=$3 - echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${FAILED_VERSION}..." - PREVIOUS_VERSION=$(get_previous_version "${REPO_URL}") - echo -e "\nAttempting to install ${PREVIOUS_VERSION}" - echo "The installed version: ${PREVIOUS_VERSION#v}" +install_previous_version() { + local given_version=$1 + local requested_version=${!given_version} + local PKG_NAME=$2 + local REPO_URL=$3 + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + requested_version=$(get_previous_version "${REPO_URL}") + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" INSTALLER_FN="install_${PKG_NAME}" - $INSTALLER_FN "${PREVIOUS_VERSION#v}" + $INSTALLER_FN "${requested_version#v}" + echo "${given_version}=${!given_version}" } install_tfdocs() { @@ -63,8 +65,7 @@ try_install_terraform_docs_dummy_version() { echo "(*) Downloading Terraform docs... ${tfdocs_filename}" install_tfdocs "$TERRAFORM_DOCS_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${tfdocs_filename}"; then - TERRAFORM_DOCS_VERSION=$(install_prev_vers "tfdocs" "${TERRAFORM_DOCS_VERSION}" "https://api.github.com/repos/terraform-docs/terraform-docs/releases" | grep "The installed version"); - TERRAFORM_DOCS_VERSION=$(echo "${TERRAFORM_DOCS_VERSION}" | sed 's/The installed version: //'); + install_previous_version TERRAFORM_DOCS_VERSION "tfdocs" "https://api.github.com/repos/terraform-docs/terraform-docs/releases" tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" fi if [ "${TERRAFORM_DOCS_SHA256}" != "dev-mode" ]; then diff --git a/test/terraform/tfsec_fallback_test.sh b/test/terraform/tfsec_fallback_test.sh index a17d00e25..eb043fff9 100644 --- a/test/terraform/tfsec_fallback_test.sh +++ b/test/terraform/tfsec_fallback_test.sh @@ -33,19 +33,21 @@ check "tfsec version as installed by feature" tfsec --version # Function to fetch the version released prior to the latest version get_previous_version() { REPO_URL=$1 - curl -s "${REPO_URL}/latest" | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/' + curl -s "${REPO_URL}/latest" | jq -r '.tag_name' } -install_prev_vers() { - PKG_NAME=$1 - FAILED_VERSION=$2 - REPO_URL=$3 - echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${FAILED_VERSION}..." - PREVIOUS_VERSION=$(get_previous_version "${REPO_URL}") - echo -e "\nAttempting to install ${PREVIOUS_VERSION}" - echo "The installed version: ${PREVIOUS_VERSION#v}" +install_previous_version() { + local given_version=$1 + local requested_version=${!given_version} + local PKG_NAME=$2 + local REPO_URL=$3 + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + requested_version=$(get_previous_version "${REPO_URL}") + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" INSTALLER_FN="install_${PKG_NAME}" - $INSTALLER_FN "${PREVIOUS_VERSION#v}" + $INSTALLER_FN "${requested_version#v}" + echo "${given_version}=${!given_version}" } install_tfsec() { @@ -63,8 +65,7 @@ try_install_tfsec_dummy_version() { echo "(*) Downloading TFSec... ${tfsec_filename}" install_tfsec "$TFSEC_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${tfsec_filename}"; then - TFSEC_VERSION=$(install_prev_vers "tfsec" "${TFSEC_VERSION}" "https://api.github.com/repos/aquasecurity/tfsec/releases" | grep "The installed version"); - TFSEC_VERSION=$(echo "${TFSEC_VERSION}" | sed 's/The installed version: //'); + install_previous_version TFSEC_VERSION "tfsec" "https://api.github.com/repos/aquasecurity/tfsec/releases" tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" fi if [ "${TFSEC_SHA256}" != "dev-mode" ]; then From b8cbd52e43a0a5ef38635a7c8fce8f70d26e867e Mon Sep 17 00:00:00 2001 From: gauravsaini04 <147703805+gauravsaini04@users.noreply.github.com> Date: Sun, 17 Mar 2024 14:29:49 +0000 Subject: [PATCH 06/21] misc change --- src/terraform/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/terraform/install.sh b/src/terraform/install.sh index 734e302a5..9be1d74b2 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -220,7 +220,7 @@ ensure_cosign() { LATEST_COSIGN_VERSION="latest" find_version_from_git_tags LATEST_COSIGN_VERSION 'https://github.com/sigstore/cosign' INSTALL_STATUS=$(install_cosign "${LATEST_COSIGN_VERSION}"); - if [[ "${INSTALL_STATUS}"=="Installation Failed." ]]; then + if [[ "${INSTALL_STATUS}" == "Installation Failed." ]]; then install_previous_version LATEST_COSIGN_VERSION "cosign" "https://api.github.com/repos/sigstore/cosign/releases" fi fi From 4860462176efb4e372ab9b6c3be02657e274b71a Mon Sep 17 00:00:00 2001 From: gauravsaini04 <147703805+gauravsaini04@users.noreply.github.com> Date: Sun, 17 Mar 2024 14:43:53 +0000 Subject: [PATCH 07/21] misc change --- src/terraform/install.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/terraform/install.sh b/src/terraform/install.sh index 9be1d74b2..cecc1422d 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -180,7 +180,11 @@ check_packages() { # Function to fetch the version released prior to the latest version get_previous_version() { REPO_URL=$1 - curl -s "${REPO_URL}/latest" | jq -r '.tag_name' + if command -v jq &> /dev/null; then + curl -s "${REPO_URL}/latest" | jq -r '.tag_name' + else + curl -s "${REPO_URL}/latest" | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/' + fi } install_previous_version() { From 9a93a40d2cae4f4debcd500f0c247703edbdba74 Mon Sep 17 00:00:00 2001 From: Gaurav Saini Date: Wed, 27 Mar 2024 02:53:11 +0000 Subject: [PATCH 08/21] 2-step fallback implemented in install.sh, test cases yet to be updated --- src/terraform/install.sh | 165 ++++++++++++++++++++++++++-------- test/terraform/scenarios.json | 7 +- 2 files changed, 132 insertions(+), 40 deletions(-) diff --git a/src/terraform/install.sh b/src/terraform/install.sh index cecc1422d..b8381bce0 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -137,6 +137,47 @@ find_version_from_git_tags() { echo "${variable_name}=${!variable_name}" } +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + find_sentinel_version_from_url() { local variable_name=$1 local requested_version=${!variable_name} @@ -179,39 +220,84 @@ check_packages() { # Function to fetch the version released prior to the latest version get_previous_version() { - REPO_URL=$1 - if command -v jq &> /dev/null; then - curl -s "${REPO_URL}/latest" | jq -r '.tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + message=$(echo "$output" | jq -r '.message') + + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" else - curl -s "${REPO_URL}/latest" | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/' - fi + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +get_pkg_name() { + local input_string="$1" + local lowercase_input="${input_string,,}" # Convert to lowercase + local suffix="_version" + local substring="${lowercase_input%$suffix*}" # Remove suffix and everything after it + echo "$substring" } install_previous_version() { - local given_version=$1 - local requested_version=${!given_version} - local PKG_NAME=$2 - local REPO_URL=$3 + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." - requested_version=$(get_previous_version "${REPO_URL}") + get_previous_version "$URL" "$REPO_URL" requested_version echo -e "\nAttempting to install ${requested_version}" declare -g ${given_version}="${requested_version#v}" INSTALLER_FN="install_${PKG_NAME}" - $INSTALLER_FN "${requested_version#v}" + $INSTALLER_FN "${!given_version}" echo "${given_version}=${!given_version}" } +# Function to check if URL returns 404 +check_failure() { + local url="$1" + local resp_code=$2 + local response_code=$(curl -o /dev/null -s -w "%{http_code}\n" "$url") + declare -g ${resp_code}="$response_code" +} + install_cosign() { - local COSIGN_VERSION=$1 - curl -L "https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" -o /tmp/cosign_${COSIGN_VERSION}_${architecture}.deb - output=$(cat /tmp/cosign_${COSIGN_VERSION}_${architecture}.deb) - if [[ "$output" != *"Not Found"* ]]; then - dpkg -i /tmp/cosign_${COSIGN_VERSION}_${architecture}.deb - rm /tmp/cosign_${COSIGN_VERSION}_${architecture}.deb - echo "Installation of cosign succeeded with ${COSIGN_VERSION}." + COSIGN_VERSION=$1 + local URL=$2 + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + resp_code=200 + check_failure "$cosign_url" resp_code + if [ "$resp_code" -eq 404 ] || [ "$resp_code" -eq 302 ]; then + echo -e "\n(!) Failed to fetch the latest artifacts for cosign v${COSIGN_VERSION}..." + REPO_URL=$(get_github_api_repo_url "$URL") + get_previous_version "$URL" "$REPO_URL" COSIGN_VERSION + echo -e "\nAttempting to install ${COSIGN_VERSION}" + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + curl -fsSL "${cosign_url}" -o $cosign_filename else - echo "Installation Failed." + curl -fsSL "${cosign_url}" -o $cosign_filename fi + dpkg -i $cosign_filename + rm $cosign_filename + echo "Installation of cosign succeeded with ${COSIGN_VERSION}." } # Install 'cosign' for validating signatures @@ -221,12 +307,10 @@ ensure_cosign() { if ! type cosign > /dev/null 2>&1; then echo "Installing cosign..." - LATEST_COSIGN_VERSION="latest" - find_version_from_git_tags LATEST_COSIGN_VERSION 'https://github.com/sigstore/cosign' - INSTALL_STATUS=$(install_cosign "${LATEST_COSIGN_VERSION}"); - if [[ "${INSTALL_STATUS}" == "Installation Failed." ]]; then - install_previous_version LATEST_COSIGN_VERSION "cosign" "https://api.github.com/repos/sigstore/cosign/releases" - fi + COSIGN_VERSION="latest" + cosign_url='https://github.com/sigstore/cosign' + find_version_from_git_tags COSIGN_VERSION "${cosign_url}" + install_cosign "${COSIGN_VERSION}" "${cosign_url}" fi if ! type cosign > /dev/null 2>&1; then echo "(!) Failed to install cosign." @@ -244,10 +328,13 @@ if ! type git > /dev/null 2>&1; then check_packages git fi +terraform_url='https://github.com/hashicorp/terraform' +tflint_url='https://github.com/terraform-linters/tflint' +terragrunt_url='https://github.com/gruntwork-io/terragrunt' # Verify requested version is available, convert latest -find_version_from_git_tags TERRAFORM_VERSION 'https://github.com/hashicorp/terraform' -find_version_from_git_tags TFLINT_VERSION 'https://github.com/terraform-linters/tflint' -find_version_from_git_tags TERRAGRUNT_VERSION 'https://github.com/gruntwork-io/terragrunt' +find_version_from_git_tags TERRAFORM_VERSION "$terraform_url" +find_version_from_git_tags TFLINT_VERSION "$tflint_url" +find_version_from_git_tags TERRAGRUNT_VERSION "$terragrunt_url" install_terraform() { local TERRAFORM_VERSION=$1 @@ -262,7 +349,7 @@ echo "Downloading terraform..." terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" install_terraform "$TERRAFORM_VERSION" if grep -q "The specified key does not exist." "${terraform_filename}"; then - install_previous_version TERRAFORM_VERSION "terraform" "https://api.github.com/repos/hashicorp/terraform/releases" + install_previous_version TERRAFORM_VERSION $terraform_url terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" fi if [ "${TERRAFORM_SHA256}" != "dev-mode" ]; then @@ -281,7 +368,6 @@ mv -f terraform /usr/local/bin/ install_tflint() { TFLINT_VERSION=$1 - TFLINT_FILENAME="tflint_linux_${architecture}.zip" curl -sSL -o /tmp/tf-downloads/${TFLINT_FILENAME} https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/${TFLINT_FILENAME} } @@ -290,7 +376,7 @@ if [ "${TFLINT_VERSION}" != "none" ]; then TFLINT_FILENAME="tflint_linux_${architecture}.zip" install_tflint "$TFLINT_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${TFLINT_FILENAME}"; then - install_previous_version TFLINT_VERSION "tflint" "https://api.github.com/repos/terraform-linters/tflint/releases" + install_previous_version TFLINT_VERSION "$tflint_url" fi if [ "${TFLINT_SHA256}" != "dev-mode" ]; then @@ -335,7 +421,6 @@ fi install_terragrunt() { TERRAGRUNT_VERSION=$1 - terragrunt_filename="terragrunt_linux_${architecture}" curl -sSL -o /tmp/tf-downloads/${terragrunt_filename} https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/${terragrunt_filename} } @@ -344,7 +429,7 @@ if [ "${TERRAGRUNT_VERSION}" != "none" ]; then terragrunt_filename="terragrunt_linux_${architecture}" install_terragrunt "$TERRAGRUNT_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${terragrunt_filename}"; then - install_previous_version TERRAGRUNT_VERSION "terragrunt" "https://api.github.com/repos/gruntwork-io/terragrunt/releases" + install_previous_version TERRAGRUNT_VERSION $terragrunt_url fi if [ "${TERRAGRUNT_SHA256}" != "dev-mode" ]; then if [ "${TERRAGRUNT_SHA256}" = "automatic" ]; then @@ -391,12 +476,13 @@ install_tfsec() { if [ "${INSTALL_TFSEC}" = "true" ]; then TFSEC_VERSION="latest" - find_version_from_git_tags TFSEC_VERSION 'https://github.com/aquasecurity/tfsec' + tfsec_url='https://github.com/aquasecurity/tfsec' + find_version_from_git_tags TFSEC_VERSION $tfsec_url tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" echo "(*) Downloading TFSec... ${tfsec_filename}" install_tfsec "$TFSEC_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${tfsec_filename}"; then - install_previous_version TFSEC_VERSION "tfsec" "https://api.github.com/repos/aquasecurity/tfsec/releases" + install_previous_version TFSEC_VERSION $tfsec_url tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" fi if [ "${TFSEC_SHA256}" != "dev-mode" ]; then @@ -413,7 +499,7 @@ if [ "${INSTALL_TFSEC}" = "true" ]; then mv -f /tmp/tf-downloads/tfsec/tfsec /usr/local/bin/tfsec fi -install_tfdocs() { +install_terraform_docs() { local TERRAFORM_DOCS_VERSION=$1 tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" curl -sSL -o /tmp/tf-downloads/${tfdocs_filename} https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/${tfdocs_filename} @@ -421,12 +507,13 @@ install_tfdocs() { if [ "${INSTALL_TERRAFORM_DOCS}" = "true" ]; then TERRAFORM_DOCS_VERSION="latest" - find_version_from_git_tags TERRAFORM_DOCS_VERSION 'https://github.com/terraform-docs/terraform-docs' + terraform_docs_url='https://github.com/terraform-docs/terraform-docs' + find_version_from_git_tags TERRAFORM_DOCS_VERSION $terraform_docs_url tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" echo "(*) Downloading Terraform docs... ${tfdocs_filename}" - install_tfdocs "$TERRAFORM_DOCS_VERSION" + install_terraform_docs "$TERRAFORM_DOCS_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${tfdocs_filename}"; then - install_previous_version TERRAFORM_DOCS_VERSION "tfdocs" "https://api.github.com/repos/terraform-docs/terraform-docs/releases" + install_previous_version TERRAFORM_DOCS_VERSION $terraform_docs_url tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" fi if [ "${TERRAFORM_DOCS_SHA256}" != "dev-mode" ]; then diff --git a/test/terraform/scenarios.json b/test/terraform/scenarios.json index 299e97cff..c7fee1ab9 100644 --- a/test/terraform/scenarios.json +++ b/test/terraform/scenarios.json @@ -35,7 +35,12 @@ "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { "terraform": { - "installTerraformDocs": true + "installTerraformDocs": true, + "terraformVersion": "latest", + "terragruntVersion": "latest", + "tflintVersion": "latest", + "installTFsec": true, + "installSentinel": true } } }, From 3e6492539bf477f22a5a9e892d055111e1feddab Mon Sep 17 00:00:00 2001 From: Gaurav Saini Date: Wed, 27 Mar 2024 04:17:24 +0000 Subject: [PATCH 09/21] install jq for ubuntu:focal, jammy & debian:11, 12 --- src/terraform/install.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/terraform/install.sh b/src/terraform/install.sh index b8381bce0..71d72824f 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -226,6 +226,12 @@ get_previous_version() { prev_version=${!variable_name} output=$(curl -s "$repo_url"); + # checking if jq package exists + if ! command -v jq &> /dev/null + then + echo "jq could not be found, attempting to install..." + apt-get update && apt-get install -y jq + fi message=$(echo "$output" | jq -r '.message') if [[ $message == "API rate limit exceeded"* ]]; then @@ -233,7 +239,7 @@ get_previous_version() { echo -e "\nAttempting to find latest version using GitHub tags." find_prev_version_from_git_tags prev_version "$url" "tags/v" declare -g ${variable_name}="${prev_version}" - else + else echo -e "\nAttempting to find latest version using GitHub Api." version=$(echo "$output" | jq -r '.tag_name') declare -g ${variable_name}="${version#v}" From 8eb282a82c91b234f1fac5563ac072e243ed34a1 Mon Sep 17 00:00:00 2001 From: Gaurav Saini Date: Wed, 27 Mar 2024 04:47:42 +0000 Subject: [PATCH 10/21] test for terraform_docs updated --- .../terraform/terraform_docs_fallback_test.sh | 156 ++++++++++++++++-- 1 file changed, 139 insertions(+), 17 deletions(-) diff --git a/test/terraform/terraform_docs_fallback_test.sh b/test/terraform/terraform_docs_fallback_test.sh index 4a4964335..5523262d5 100644 --- a/test/terraform/terraform_docs_fallback_test.sh +++ b/test/terraform/terraform_docs_fallback_test.sh @@ -30,42 +30,163 @@ case ${architecture} in *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; esac +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + # Function to fetch the version released prior to the latest version get_previous_version() { - REPO_URL=$1 - curl -s "${REPO_URL}/latest" | jq -r '.tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + # checking if jq package exists + if ! command -v jq &> /dev/null + then + echo "jq could not be found, attempting to install..." + apt-get update && apt-get install -y jq + fi + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +get_pkg_name() { + local input_string="$1" + local lowercase_input="${input_string,,}" # Convert to lowercase + local suffix="_version" + local substring="${lowercase_input%$suffix*}" # Remove suffix and everything after it + echo "$substring" } install_previous_version() { - local given_version=$1 - local requested_version=${!given_version} - local PKG_NAME=$2 - local REPO_URL=$3 + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." - requested_version=$(get_previous_version "${REPO_URL}") + get_previous_version "$URL" "$REPO_URL" requested_version $mode echo -e "\nAttempting to install ${requested_version}" declare -g ${given_version}="${requested_version#v}" INSTALLER_FN="install_${PKG_NAME}" - $INSTALLER_FN "${requested_version#v}" + $INSTALLER_FN "${!given_version}" echo "${given_version}=${!given_version}" } -install_tfdocs() { - TERRAFORM_DOCS_VERSION=$1 +install_terraform_docs() { + local TERRAFORM_DOCS_VERSION=$1 tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" curl -sSL -o /tmp/tf-downloads/${tfdocs_filename} https://github.com/terraform-docs/terraform-docs/releases/download/v${TERRAFORM_DOCS_VERSION}/${tfdocs_filename} } + try_install_terraform_docs_dummy_version() { + mode=$1 mkdir -p /tmp/tf-downloads cd /tmp/tf-downloads - TERRAFORM_DOCS_VERSION="1.2.xyz" + TERRAFORM_DOCS_VERSION="0.17.xyz" echo -e "\nInstalling TERRAFORM_DOCS dummy version.." v${TERRAFORM_DOCS_VERSION} + terraform_docs_url='https://github.com/terraform-docs/terraform-docs' tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" echo "(*) Downloading Terraform docs... ${tfdocs_filename}" - install_tfdocs "$TERRAFORM_DOCS_VERSION" + install_terraform_docs "$TERRAFORM_DOCS_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${tfdocs_filename}"; then - install_previous_version TERRAFORM_DOCS_VERSION "tfdocs" "https://api.github.com/repos/terraform-docs/terraform-docs/releases" + install_previous_version TERRAFORM_DOCS_VERSION $terraform_docs_url $mode tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" fi if [ "${TERRAFORM_DOCS_SHA256}" != "dev-mode" ]; then @@ -80,14 +201,15 @@ try_install_terraform_docs_dummy_version() { tar -xzf /tmp/tf-downloads/${tfdocs_filename} -C /tmp/tf-downloads/tfdocs sudo chmod a+x /tmp/tf-downloads/tfdocs/terraform-docs sudo mv -f /tmp/tf-downloads/tfdocs/terraform-docs /usr/local/bin/terraform-docs - - rm -rf /tmp/tf-downloads ${GNUPGHOME} } -try_install_terraform_docs_dummy_version +try_install_terraform_docs_dummy_version "mode1" + +check "terraform-docs version as installed by test (mode 1: install using find_prev_version_from_git_tags)" terraform-docs --version +try_install_terraform_docs_dummy_version "mode2" -check "terraform-docs version as installed by test" terraform-docs --version +check "terraform-docs version as installed by test (mode 2: install using GitHub Api)" terraform-docs --version # Report result reportResults \ No newline at end of file From 2dbf47e0fc17deeb20790731278f36f9e6e17052 Mon Sep 17 00:00:00 2001 From: Gaurav Saini Date: Wed, 27 Mar 2024 04:56:05 +0000 Subject: [PATCH 11/21] tfsec test updated --- test/terraform/tfsec_fallback_test.sh | 150 +++++++++++++++++++++++--- 1 file changed, 137 insertions(+), 13 deletions(-) diff --git a/test/terraform/tfsec_fallback_test.sh b/test/terraform/tfsec_fallback_test.sh index eb043fff9..208bbbe40 100644 --- a/test/terraform/tfsec_fallback_test.sh +++ b/test/terraform/tfsec_fallback_test.sh @@ -30,42 +30,162 @@ esac # TFSec specific tests check "tfsec version as installed by feature" tfsec --version +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + # Function to fetch the version released prior to the latest version get_previous_version() { - REPO_URL=$1 - curl -s "${REPO_URL}/latest" | jq -r '.tag_name' + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + # checking if jq package exists + if ! command -v jq &> /dev/null + then + echo "jq could not be found, attempting to install..." + apt-get update && apt-get install -y jq + fi + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +get_pkg_name() { + local input_string="$1" + local lowercase_input="${input_string,,}" # Convert to lowercase + local suffix="_version" + local substring="${lowercase_input%$suffix*}" # Remove suffix and everything after it + echo "$substring" } install_previous_version() { - local given_version=$1 - local requested_version=${!given_version} - local PKG_NAME=$2 - local REPO_URL=$3 + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." - requested_version=$(get_previous_version "${REPO_URL}") + get_previous_version "$URL" "$REPO_URL" requested_version $mode echo -e "\nAttempting to install ${requested_version}" declare -g ${given_version}="${requested_version#v}" INSTALLER_FN="install_${PKG_NAME}" - $INSTALLER_FN "${requested_version#v}" + $INSTALLER_FN "${!given_version}" echo "${given_version}=${!given_version}" } install_tfsec() { - TFSEC_VERSION=$1 + local TFSEC_VERSION=$1 tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" curl -sSL -o /tmp/tf-downloads/${tfsec_filename} https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${tfsec_filename} } try_install_tfsec_dummy_version() { + mode=$1 mkdir -p /tmp/tf-downloads cd /tmp/tf-downloads - TFSEC_VERSION="1.2.xyz" + TFSEC_VERSION="1.28.XYZ" echo -e "\nInstalling TFSEC dummy version.." v${TFSEC_VERSION} + tfsec_url='https://github.com/aquasecurity/tfsec' tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" echo "(*) Downloading TFSec... ${tfsec_filename}" install_tfsec "$TFSEC_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${tfsec_filename}"; then - install_previous_version TFSEC_VERSION "tfsec" "https://api.github.com/repos/aquasecurity/tfsec/releases" + install_previous_version TFSEC_VERSION $tfsec_url $mode tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" fi if [ "${TFSEC_SHA256}" != "dev-mode" ]; then @@ -82,9 +202,13 @@ try_install_tfsec_dummy_version() { sudo mv -f /tmp/tf-downloads/tfsec/tfsec /usr/local/bin/tfsec } -try_install_tfsec_dummy_version +try_install_tfsec_dummy_version "mode1" + +check "tfsec version as installed by test after fallbacking from the dummy version (mode 1: install using find_prev_version_from_git_tags)" tfsec --version + +try_install_tfsec_dummy_version "mode2" -check "tfsec version as installed by test after fallbacking from the dummy version" tfsec --version +check "tfsec version as installed by test after fallbacking from the dummy version (mode 2: install using GitHub Api)" tfsec --version # Report result reportResults \ No newline at end of file From c2715ab8853734821cffcd77ed36215290e1169a Mon Sep 17 00:00:00 2001 From: Gaurav Saini Date: Wed, 27 Mar 2024 05:20:38 +0000 Subject: [PATCH 12/21] terraform fallback test updated --- test/terraform/scenarios.json | 15 +- test/terraform/terraform_fallback_test.sh | 258 ++++++++++++++++++++++ 2 files changed, 267 insertions(+), 6 deletions(-) create mode 100644 test/terraform/terraform_fallback_test.sh diff --git a/test/terraform/scenarios.json b/test/terraform/scenarios.json index c7fee1ab9..3f8a4bd8d 100644 --- a/test/terraform/scenarios.json +++ b/test/terraform/scenarios.json @@ -35,12 +35,15 @@ "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { "terraform": { - "installTerraformDocs": true, - "terraformVersion": "latest", - "terragruntVersion": "latest", - "tflintVersion": "latest", - "installTFsec": true, - "installSentinel": true + "installTerraformDocs": true + } + } + }, + "terraform_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "terraformVersion": "latest" } } }, diff --git a/test/terraform/terraform_fallback_test.sh b/test/terraform/terraform_fallback_test.sh new file mode 100644 index 000000000..2a14310d7 --- /dev/null +++ b/test/terraform/terraform_fallback_test.sh @@ -0,0 +1,258 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +handle_error() { + echo "Error occurred on line ${BASH_LINENO[0]}" + exit 1 +} + +# Trap errors and call the error handling function +trap 'handle_error' ERR + +check "terraform version as installed by feature" terraform --version + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Import the specified key in a variable name passed in as +receive_gpg_keys() { + local keys=${!1} + local keyring_args="" + if [ ! -z "$2" ]; then + keyring_args="--no-default-keyring --keyring $2" + fi + if [ ! -z "${KEYSERVER_PROXY}" ]; then + keyring_args="${keyring_args} --keyserver-options http-proxy=${KEYSERVER_PROXY}" + fi + + # Use a temporary location for gpg keys to avoid polluting image + export GNUPGHOME="/tmp/tmp-gnupg" + mkdir -p ${GNUPGHOME} + chmod 700 ${GNUPGHOME} + echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" > ${GNUPGHOME}/dirmngr.conf + # GPG key download sometimes fails for some reason and retrying fixes it. + local retry_count=0 + local gpg_ok="false" + set +e + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; + do + echo "(*) Downloading GPG key..." + ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" + if [ "${gpg_ok}" != "true" ]; then + echo "(*) Failed getting key, retring in 10s..." + (( retry_count++ )) + sleep 10s + fi + done + + # If all attempts fail, try getting the keyserver IP address and explicitly passing it to gpg + if [ "${gpg_ok}" = "false" ]; then + retry_count=0; + echo "(*) Resolving GPG keyserver IP address..." + local keyserver_ip_address=$( dig +short keyserver.ubuntu.com | head -n1 ) + echo "(*) GPG keyserver IP address $keyserver_ip_address" + + until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "3" ]; + do + echo "(*) Downloading GPG key..." + ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys --keyserver ${keyserver_ip_address}) 2>&1 && gpg_ok="true" + if [ "${gpg_ok}" != "true" ]; then + echo "(*) Failed getting key, retring in 10s..." + (( retry_count++ )) + sleep 10s + fi + done + fi + set -e + if [ "${gpg_ok}" = "false" ]; then + echo "(!) Failed to get gpg key." + exit 1 + fi +} + + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + # checking if jq package exists + if ! command -v jq &> /dev/null + then + echo "jq could not be found, attempting to install..." + apt-get update && apt-get install -y jq + fi + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +get_pkg_name() { + local input_string="$1" + local lowercase_input="${input_string,,}" # Convert to lowercase + local suffix="_version" + local substring="${lowercase_input%$suffix*}" # Remove suffix and everything after it + echo "$substring" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + INSTALLER_FN="install_${PKG_NAME}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +install_terraform() { + local TERRAFORM_VERSION=$1 + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + curl -sSL -o ${terraform_filename} "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/${terraform_filename}" +} + +try_install_dummy_terraform_version() { + mode=$1 + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + terraform_url='https://github.com/hashicorp/terraform' + TERRAFORM_VERSION="1.7.xyz" + echo -e "\nAttempting to install dummy version for Terraform v${TERRAFORM_VERSION}..." + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + install_terraform "$TERRAFORM_VERSION" + if grep -q "The specified key does not exist." "${terraform_filename}"; then + install_previous_version TERRAFORM_VERSION $terraform_url $mode + terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" + fi + unzip ${terraform_filename} + sudo mv -f terraform /usr/local/bin/ +} + +try_install_dummy_terraform_version "mode1" + +check "terraform version as installed by test after fallbacking from the dummy version (mode 1: install using find_prev_version_from_git_tags)" terraform --version + +try_install_dummy_terraform_version "mode2" + +check "terraform version as installed by test after fallbacking from the dummy version (mode 2: install using GitHub Api)" terraform --version + +# Report result +reportResults + From d639c4d8e56705d37111e61bbcf3eb7d8c989f61 Mon Sep 17 00:00:00 2001 From: Gaurav Saini Date: Wed, 27 Mar 2024 05:24:46 +0000 Subject: [PATCH 13/21] misc change --- test/terraform/terraform_fallback_test.sh | 57 ----------------------- 1 file changed, 57 deletions(-) diff --git a/test/terraform/terraform_fallback_test.sh b/test/terraform/terraform_fallback_test.sh index 2a14310d7..b69e0083c 100644 --- a/test/terraform/terraform_fallback_test.sh +++ b/test/terraform/terraform_fallback_test.sh @@ -27,63 +27,6 @@ case ${architecture} in *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; esac -# Import the specified key in a variable name passed in as -receive_gpg_keys() { - local keys=${!1} - local keyring_args="" - if [ ! -z "$2" ]; then - keyring_args="--no-default-keyring --keyring $2" - fi - if [ ! -z "${KEYSERVER_PROXY}" ]; then - keyring_args="${keyring_args} --keyserver-options http-proxy=${KEYSERVER_PROXY}" - fi - - # Use a temporary location for gpg keys to avoid polluting image - export GNUPGHOME="/tmp/tmp-gnupg" - mkdir -p ${GNUPGHOME} - chmod 700 ${GNUPGHOME} - echo -e "disable-ipv6\n${GPG_KEY_SERVERS}" > ${GNUPGHOME}/dirmngr.conf - # GPG key download sometimes fails for some reason and retrying fixes it. - local retry_count=0 - local gpg_ok="false" - set +e - until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "5" ]; - do - echo "(*) Downloading GPG key..." - ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys) 2>&1 && gpg_ok="true" - if [ "${gpg_ok}" != "true" ]; then - echo "(*) Failed getting key, retring in 10s..." - (( retry_count++ )) - sleep 10s - fi - done - - # If all attempts fail, try getting the keyserver IP address and explicitly passing it to gpg - if [ "${gpg_ok}" = "false" ]; then - retry_count=0; - echo "(*) Resolving GPG keyserver IP address..." - local keyserver_ip_address=$( dig +short keyserver.ubuntu.com | head -n1 ) - echo "(*) GPG keyserver IP address $keyserver_ip_address" - - until [ "${gpg_ok}" = "true" ] || [ "${retry_count}" -eq "3" ]; - do - echo "(*) Downloading GPG key..." - ( echo "${keys}" | xargs -n 1 gpg -q ${keyring_args} --recv-keys --keyserver ${keyserver_ip_address}) 2>&1 && gpg_ok="true" - if [ "${gpg_ok}" != "true" ]; then - echo "(*) Failed getting key, retring in 10s..." - (( retry_count++ )) - sleep 10s - fi - done - fi - set -e - if [ "${gpg_ok}" = "false" ]; then - echo "(!) Failed to get gpg key." - exit 1 - fi -} - - # Figure out correct version of a three part version number is not passed find_version_from_git_tags() { local variable_name=$1 From 2f701d6838612b4c5e1928e9eb75c7284ba08876 Mon Sep 17 00:00:00 2001 From: Gaurav Saini Date: Wed, 27 Mar 2024 06:09:32 +0000 Subject: [PATCH 14/21] added tflint fallback test file --- src/terraform/install.sh | 4 +- test/terraform/scenarios.json | 8 + test/terraform/tflint_fallback_test.sh | 293 +++++++++++++++++++++++++ 3 files changed, 303 insertions(+), 2 deletions(-) create mode 100644 test/terraform/tflint_fallback_test.sh diff --git a/src/terraform/install.sh b/src/terraform/install.sh index 71d72824f..c40b1908b 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -297,9 +297,9 @@ install_cosign() { echo -e "\nAttempting to install ${COSIGN_VERSION}" cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" - curl -fsSL "${cosign_url}" -o $cosign_filename + curl -L "${cosign_url}" -o $cosign_filename else - curl -fsSL "${cosign_url}" -o $cosign_filename + curl -L "${cosign_url}" -o $cosign_filename fi dpkg -i $cosign_filename rm $cosign_filename diff --git a/test/terraform/scenarios.json b/test/terraform/scenarios.json index 3f8a4bd8d..0c69fb343 100644 --- a/test/terraform/scenarios.json +++ b/test/terraform/scenarios.json @@ -47,6 +47,14 @@ } } }, + "tflint_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "tflintVersion": "latest" + } + } + }, "older_tflint": { "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { diff --git a/test/terraform/tflint_fallback_test.sh b/test/terraform/tflint_fallback_test.sh new file mode 100644 index 000000000..5a9042dfb --- /dev/null +++ b/test/terraform/tflint_fallback_test.sh @@ -0,0 +1,293 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +handle_error() { + echo "Error occurred on line ${BASH_LINENO[0]}" + exit 1 +} + +# Trap errors and call the error handling function +trap 'handle_error' ERR + +TFLINT_SHA256="automatic" + +GPG_KEY_SERVERS="keyserver hkps://keyserver.ubuntu.com +keyserver hkps://keys.openpgp.org +keyserver hkps://keyserver.pgp.com" + +check "tflint version as installed by feature" tflint --version +check "cosign version as installed by feature" cosign version + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + # checking if jq package exists + if ! command -v jq &> /dev/null + then + echo "jq could not be found, attempting to install..." + apt-get update && apt-get install -y jq + fi + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +get_pkg_name() { + local input_string="$1" + local lowercase_input="${input_string,,}" # Convert to lowercase + local suffix="_version" + local substring="${lowercase_input%$suffix*}" # Remove suffix and everything after it + echo "$substring" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + INSTALLER_FN="install_${PKG_NAME}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + +# Function to check if URL returns 404 +check_failure() { + local url="$1" + local resp_code=$2 + local response_code=$(curl -o /dev/null -s -w "%{http_code}\n" "$url") + declare -g ${resp_code}="$response_code" +} + +install_cosign() { + COSIGN_VERSION=$1 + local URL=$2 + local mode=$3 + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + resp_code=200 + check_failure "$cosign_url" resp_code + if [ "$resp_code" -eq 404 ] || [ "$resp_code" -eq 302 ]; then + echo -e "\n(!) Failed to fetch the latest artifacts for cosign v${COSIGN_VERSION}..." + REPO_URL=$(get_github_api_repo_url "$URL") + get_previous_version "$URL" "$REPO_URL" COSIGN_VERSION $mode + echo -e "\nAttempting to install ${COSIGN_VERSION}" + cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" + cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" + curl -L "${cosign_url}" -o $cosign_filename + else + curl -L "${cosign_url}" -o $cosign_filename + fi + dpkg -i $cosign_filename + rm $cosign_filename + echo "Installation of cosign succeeded with ${COSIGN_VERSION}." +} + +# Install 'cosign' for validating signatures +# https://docs.sigstore.dev/cosign/overview/ +ensure_cosign() { + mode=$1 + if ! type cosign > /dev/null 2>&1; then + echo -e "\nAttempting to install dummy cosign version..." + COSIGN_VERSION="2.2.xyz" + echo "Installing cosign... v${COSIGN_VERSION}" + cosign_url='https://github.com/sigstore/cosign' + install_cosign "${COSIGN_VERSION}" "${cosign_url}" $mode + fi + if ! type cosign > /dev/null 2>&1; then + echo "(!) Failed to install cosign." + exit 1 + fi + cosign version +} + +install_tflint() { + TFLINT_VERSION=$1 + curl -sSL -o /tmp/tf-downloads/${TFLINT_FILENAME} https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/${TFLINT_FILENAME} +} + + +try_install_dummy_tflint_cosign_version() { + mode=$1 + tflint_url='https://github.com/terraform-linters/tflint' + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + echo -e "\nTrying to install dummy tflint version..." + TFLINT_VERSION="0.50.XYZ" + echo "Downloading tflint...v${TFLINT_VERSION}" + TFLINT_FILENAME="tflint_linux_${architecture}.zip" + install_tflint "$TFLINT_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${TFLINT_FILENAME}"; then + install_previous_version TFLINT_VERSION "$tflint_url" $mode + fi + if [ "${TFLINT_SHA256}" != "dev-mode" ]; then + + if [ "${TFLINT_SHA256}" != "automatic" ]; then + echo "${TFLINT_SHA256} *${TFLINT_FILENAME}" > tflint_checksums.txt + sha256sum --ignore-missing -c tflint_checksums.txt + else + curl -sSL -o tflint_checksums.txt https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt + + set +e + curl -sSL -o checksums.txt.keyless.sig https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt.keyless.sig + set -e + + # Check that checksums.txt.keyless.sig exists and is not empty + if [ -s checksums.txt.keyless.sig ]; then + # Validate checksums with cosign + curl -sSL -o checksums.txt.pem https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt.pem + ensure_cosign $mode + cosign verify-blob \ + --certificate=/tmp/tf-downloads/checksums.txt.pem \ + --signature=/tmp/tf-downloads/checksums.txt.keyless.sig \ + --certificate-identity-regexp="^https://github.com/terraform-linters/tflint" \ + --certificate-oidc-issuer=https://token.actions.githubusercontent.com \ + /tmp/tf-downloads/tflint_checksums.txt + # Ensure that checksums.txt has $TFLINT_FILENAME + grep ${TFLINT_FILENAME} /tmp/tf-downloads/tflint_checksums.txt + # Validate downloaded file + sha256sum --ignore-missing -c tflint_checksums.txt + else + # Fallback to older, GPG-based verification (pre-0.47.0 of tflint) + curl -sSL -o tflint_checksums.txt.sig https://github.com/terraform-linters/tflint/releases/download/v${TFLINT_VERSION}/checksums.txt.sig + curl -sSL -o tflint_key "${TFLINT_GPG_KEY_URI}" + gpg -q --import tflint_key + gpg --verify tflint_checksums.txt.sig tflint_checksums.txt + fi + fi + fi + + unzip /tmp/tf-downloads/${TFLINT_FILENAME} + sudo mv -f tflint /usr/local/bin/ +} + +try_install_dummy_tflint_cosign_version "mode1" + +check "tflint version as installed when mode=1" tflint --version +check "cosign version as installed when mode=1" cosign version + +try_install_dummy_tflint_cosign_version "mode2" + +check "tflint version as installed when mode=2" tflint --version +check "cosign version as installed when mode=2" cosign version \ No newline at end of file From 4a983e6b40d2d32398d76a5219d21fef14e5df4f Mon Sep 17 00:00:00 2001 From: Gaurav Saini Date: Wed, 27 Mar 2024 08:28:34 +0000 Subject: [PATCH 15/21] terragrunt fallback test file added --- test/terraform/scenarios.json | 12 +- .../terraform/terraform_docs_fallback_test.sh | 9 +- test/terraform/terraform_fallback_test.sh | 9 +- test/terraform/terragrunt_fallback_test.sh | 211 ++++++++++++++++++ test/terraform/tflint_fallback_test.sh | 9 +- test/terraform/tfsec_fallback_test.sh | 8 +- 6 files changed, 237 insertions(+), 21 deletions(-) create mode 100644 test/terraform/terragrunt_fallback_test.sh diff --git a/test/terraform/scenarios.json b/test/terraform/scenarios.json index 0c69fb343..04693666c 100644 --- a/test/terraform/scenarios.json +++ b/test/terraform/scenarios.json @@ -43,7 +43,15 @@ "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { "terraform": { - "terraformVersion": "latest" + "version": "latest" + } + } + }, + "terragrunt_fallback_test": { + "image": "mcr.microsoft.com/devcontainers/base:jammy", + "features": { + "terraform": { + "terragrunt": "latest" } } }, @@ -51,7 +59,7 @@ "image": "mcr.microsoft.com/devcontainers/base:jammy", "features": { "terraform": { - "tflintVersion": "latest" + "tflint": "latest" } } }, diff --git a/test/terraform/terraform_docs_fallback_test.sh b/test/terraform/terraform_docs_fallback_test.sh index 5523262d5..ab7d03b43 100644 --- a/test/terraform/terraform_docs_fallback_test.sh +++ b/test/terraform/terraform_docs_fallback_test.sh @@ -13,13 +13,12 @@ check "terraform-docs version as installed by feature" terraform-docs --version TERRAFORM_DOCS_SHA256="automatic" -handle_error() { - echo "Error occurred on line ${BASH_LINENO[0]}" - exit 1 +set_error_handler() { + echo "Error occurred on line: $LINENO" } -# Trap errors and call the error handling function -trap 'handle_error' ERR +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR architecture="$(uname -m)" case ${architecture} in diff --git a/test/terraform/terraform_fallback_test.sh b/test/terraform/terraform_fallback_test.sh index b69e0083c..f9d34f9cf 100644 --- a/test/terraform/terraform_fallback_test.sh +++ b/test/terraform/terraform_fallback_test.sh @@ -8,13 +8,12 @@ source dev-container-features-test-lib # Check to make sure the user is vscode check "user is vscode" whoami | grep vscode -handle_error() { - echo "Error occurred on line ${BASH_LINENO[0]}" - exit 1 +set_error_handler() { + echo "Error occurred on line: $LINENO" } -# Trap errors and call the error handling function -trap 'handle_error' ERR +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR check "terraform version as installed by feature" terraform --version diff --git a/test/terraform/terragrunt_fallback_test.sh b/test/terraform/terragrunt_fallback_test.sh new file mode 100644 index 000000000..510b54d5f --- /dev/null +++ b/test/terraform/terragrunt_fallback_test.sh @@ -0,0 +1,211 @@ +#!/bin/bash + +set -e + +# Import test library for `check` command +source dev-container-features-test-lib + +# Check to make sure the user is vscode +check "user is vscode" whoami | grep vscode + +set_error_handler() { + echo "Error occurred on line: $LINENO" +} + +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR + +check "terragrunt version as installed by feature" terragrunt --version + +TERRAGRUNT_SHA256="automatic" + +architecture="$(uname -m)" +case ${architecture} in + x86_64) architecture="amd64";; + aarch64 | armv8*) architecture="arm64";; + aarch32 | armv7* | armvhf*) architecture="arm";; + i?86) architecture="386";; + *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; +esac + +# Figure out correct version of a three part version number is not passed +find_version_from_git_tags() { + local variable_name=$1 + local requested_version=${!variable_name} + if [ "${requested_version}" = "none" ]; then return; fi + local repository=$2 + local prefix=${3:-"tags/v"} + local separator=${4:-"."} + local last_part_optional=${5:-"false"} + if [ "$(echo "${requested_version}" | grep -o "." | wc -l)" != "2" ]; then + local escaped_separator=${separator//./\\.} + local last_part + if [ "${last_part_optional}" = "true" ]; then + last_part="(${escaped_separator}[0-9]+)?" + else + last_part="${escaped_separator}[0-9]+" + fi + local regex="${prefix}\\K[0-9]+${escaped_separator}[0-9]+${last_part}$" + local version_list="$(git ls-remote --tags ${repository} | grep -oP "${regex}" | tr -d ' ' | tr "${separator}" "." | sort -rV)" + if [ "${requested_version}" = "latest" ] || [ "${requested_version}" = "current" ] || [ "${requested_version}" = "lts" ]; then + declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)" + else + set +e + declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")" + set -e + fi + fi + if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then + echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2 + exit 1 + fi + echo "${variable_name}=${!variable_name}" +} + +# Use semver logic to decrement a version number then look for the closest match +find_prev_version_from_git_tags() { + local variable_name=$1 + local current_version=${!variable_name} + local repository=$2 + # Normally a "v" is used before the version number, but support alternate cases + local prefix=${3:-"tags/v"} + # Some repositories use "_" instead of "." for version number part separation, support that + local separator=${4:-"."} + # Some tools release versions that omit the last digit (e.g. go) + local last_part_optional=${5:-"false"} + # Some repositories may have tags that include a suffix (e.g. actions/node-versions) + local version_suffix_regex=$6 + # Try one break fix version number less if we get a failure. Use "set +e" since "set -e" can cause failures in valid scenarios. + set +e + major="$(echo "${current_version}" | grep -oE '^[0-9]+' || echo '')" + minor="$(echo "${current_version}" | grep -oP '^[0-9]+\.\K[0-9]+' || echo '')" + breakfix="$(echo "${current_version}" | grep -oP '^[0-9]+\.[0-9]+\.\K[0-9]+' 2>/dev/null || echo '')" + + if [ "${minor}" = "0" ] && [ "${breakfix}" = "0" ]; then + ((major=major-1)) + declare -g ${variable_name}="${major}" + # Look for latest version from previous major release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + # Handle situations like Go's odd version pattern where "0" releases omit the last part + elif [ "${breakfix}" = "" ] || [ "${breakfix}" = "0" ]; then + ((minor=minor-1)) + declare -g ${variable_name}="${major}.${minor}" + # Look for latest version from previous minor release + find_version_from_git_tags "${variable_name}" "${repository}" "${prefix}" "${separator}" "${last_part_optional}" + else + ((breakfix=breakfix-1)) + if [ "${breakfix}" = "0" ] && [ "${last_part_optional}" = "true" ]; then + declare -g ${variable_name}="${major}.${minor}" + else + declare -g ${variable_name}="${major}.${minor}.${breakfix}" + fi + fi + set -e +} + +# Function to fetch the version released prior to the latest version +get_previous_version() { + local url=$1 + local repo_url=$2 + local variable_name=$3 + local mode=$4 + prev_version=${!variable_name} + + output=$(curl -s "$repo_url"); + # checking if jq package exists + if ! command -v jq &> /dev/null + then + echo "jq could not be found, attempting to install..." + apt-get update && apt-get install -y jq + fi + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then + message="API rate limit exceeded"; + elif [[ "$mode" == "mode2" ]]; then + message="" + fi + if [[ $message == "API rate limit exceeded"* ]]; then + echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" + echo -e "\nAttempting to find latest version using GitHub tags." + find_prev_version_from_git_tags prev_version "$url" "tags/v" + declare -g ${variable_name}="${prev_version}" + else + echo -e "\nAttempting to find latest version using GitHub Api." + version=$(echo "$output" | jq -r '.tag_name') + declare -g ${variable_name}="${version#v}" + fi + echo "${variable_name}=${!variable_name}" +} + +get_github_api_repo_url() { + local url=$1 + echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" +} + +get_pkg_name() { + local input_string="$1" + local lowercase_input="${input_string,,}" # Convert to lowercase + local suffix="_version" + local substring="${lowercase_input%$suffix*}" # Remove suffix and everything after it + echo "$substring" +} + +install_previous_version() { + given_version=$1 + requested_version=${!given_version} + local URL=$2 + local mode=$3 + local REPO_URL=$(get_github_api_repo_url "$URL") + local PKG_NAME=$(get_pkg_name "${given_version}") + echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." + get_previous_version "$URL" "$REPO_URL" requested_version $mode + echo -e "\nAttempting to install ${requested_version}" + declare -g ${given_version}="${requested_version#v}" + INSTALLER_FN="install_${PKG_NAME}" + $INSTALLER_FN "${!given_version}" + echo "${given_version}=${!given_version}" +} + + +install_terragrunt() { + TERRAGRUNT_VERSION=$1 + curl -sSL -o /tmp/tf-downloads/${terragrunt_filename} https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/${terragrunt_filename} +} + + +try_install_dummy_terragrunt_version() { + mode=$1 + mkdir -p /tmp/tf-downloads + cd /tmp/tf-downloads + terragrunt_url='https://github.com/gruntwork-io/terragrunt' + TERRAGRUNT_VERSION="0.55.xyz" + echo -e "\nAttempting to install terragrunt dummy v${TERRAGRUNT_VERSION}" + echo "Downloading Terragrunt... v${TERRAGRUNT_VERSION}" + terragrunt_filename="terragrunt_linux_${architecture}" + install_terragrunt "$TERRAGRUNT_VERSION" + if grep -q "Not Found" "/tmp/tf-downloads/${terragrunt_filename}"; then + install_previous_version TERRAGRUNT_VERSION $terragrunt_url $mode + fi + if [ "${TERRAGRUNT_SHA256}" != "dev-mode" ]; then + if [ "${TERRAGRUNT_SHA256}" = "automatic" ]; then + curl -sSL -o terragrunt_SHA256SUMS https://github.com/gruntwork-io/terragrunt/releases/download/v${TERRAGRUNT_VERSION}/SHA256SUMS + else + echo "${TERRAGRUNT_SHA256} *${terragrunt_filename}" > terragrunt_SHA256SUMS + fi + sha256sum --ignore-missing -c terragrunt_SHA256SUMS + fi + sudo chmod a+x /tmp/tf-downloads/${terragrunt_filename} + sudo mv -f /tmp/tf-downloads/${terragrunt_filename} /usr/local/bin/terragrunt +} + +try_install_dummy_terragrunt_version "mode1" + +check "terragrunt version as installed by test after fallbacking from the dummy version (mode 1: install using find_prev_version_from_git_tags)" terragrunt --version + +try_install_dummy_terragrunt_version "mode2" + +check "terragrunt version as installed by test after fallbacking from the dummy version (mode 2: install using GitHub Api)" terragrunt --version + +# Report result +reportResults + diff --git a/test/terraform/tflint_fallback_test.sh b/test/terraform/tflint_fallback_test.sh index 5a9042dfb..6d43d2259 100644 --- a/test/terraform/tflint_fallback_test.sh +++ b/test/terraform/tflint_fallback_test.sh @@ -8,13 +8,12 @@ source dev-container-features-test-lib # Check to make sure the user is vscode check "user is vscode" whoami | grep vscode -handle_error() { - echo "Error occurred on line ${BASH_LINENO[0]}" - exit 1 +set_error_handler() { + echo "Error occurred on line: $LINENO" } -# Trap errors and call the error handling function -trap 'handle_error' ERR +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR TFLINT_SHA256="automatic" diff --git a/test/terraform/tfsec_fallback_test.sh b/test/terraform/tfsec_fallback_test.sh index 208bbbe40..081c7e794 100644 --- a/test/terraform/tfsec_fallback_test.sh +++ b/test/terraform/tfsec_fallback_test.sh @@ -8,12 +8,12 @@ source dev-container-features-test-lib # Check to make sure the user is vscode check "user is vscode" whoami | grep vscode -handle_error() { - echo "Error occurred on line ${BASH_LINENO[0]}" - exit 1 +set_error_handler() { + echo "Error occurred on line: $LINENO" } -TFSEC_SHA256="automatic" +# Register the error handler function to be triggered on ERR signal +trap 'set_error_handler' ERR # Trap errors and call the error handling function trap 'handle_error' ERR From cda53a1b3fde4976a59cc64b7540f07330f403aa Mon Sep 17 00:00:00 2001 From: Gaurav Saini Date: Wed, 27 Mar 2024 08:59:08 +0000 Subject: [PATCH 16/21] minor change in tfsec test file --- test/terraform/tfsec_fallback_test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/terraform/tfsec_fallback_test.sh b/test/terraform/tfsec_fallback_test.sh index 081c7e794..63d6362e1 100644 --- a/test/terraform/tfsec_fallback_test.sh +++ b/test/terraform/tfsec_fallback_test.sh @@ -27,6 +27,8 @@ case ${architecture} in *) echo "(!) Architecture ${architecture} unsupported"; exit 1 ;; esac +TFSEC_SHA256="automatic" + # TFSec specific tests check "tfsec version as installed by feature" tfsec --version From 2814ddf7afa70e39eac8f246cc859b1f4336d621 Mon Sep 17 00:00:00 2001 From: Gaurav Saini Date: Wed, 27 Mar 2024 16:12:01 +0000 Subject: [PATCH 17/21] few more changes to cosign, terragrunt installations.. --- src/terraform/install.sh | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/src/terraform/install.sh b/src/terraform/install.sh index c40b1908b..db067e98d 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -275,22 +275,13 @@ install_previous_version() { echo "${given_version}=${!given_version}" } -# Function to check if URL returns 404 -check_failure() { - local url="$1" - local resp_code=$2 - local response_code=$(curl -o /dev/null -s -w "%{http_code}\n" "$url") - declare -g ${resp_code}="$response_code" -} - install_cosign() { COSIGN_VERSION=$1 local URL=$2 cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" - resp_code=200 - check_failure "$cosign_url" resp_code - if [ "$resp_code" -eq 404 ] || [ "$resp_code" -eq 302 ]; then + curl -L "${cosign_url}" -o $cosign_filename + if grep -q "Not Found" "$cosign_filename"; then echo -e "\n(!) Failed to fetch the latest artifacts for cosign v${COSIGN_VERSION}..." REPO_URL=$(get_github_api_repo_url "$URL") get_previous_version "$URL" "$REPO_URL" COSIGN_VERSION @@ -298,8 +289,6 @@ install_cosign() { cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" curl -L "${cosign_url}" -o $cosign_filename - else - curl -L "${cosign_url}" -o $cosign_filename fi dpkg -i $cosign_filename rm $cosign_filename @@ -434,7 +423,8 @@ if [ "${TERRAGRUNT_VERSION}" != "none" ]; then echo "Downloading Terragrunt..." terragrunt_filename="terragrunt_linux_${architecture}" install_terragrunt "$TERRAGRUNT_VERSION" - if grep -q "Not Found" "/tmp/tf-downloads/${terragrunt_filename}"; then + output=$(cat "/tmp/tf-downloads/${terragrunt_filename}") + if [[ $output == "Not Found" ]]; then install_previous_version TERRAGRUNT_VERSION $terragrunt_url fi if [ "${TERRAGRUNT_SHA256}" != "dev-mode" ]; then From 815400621b93d023d0a79e592a0589c655bddf91 Mon Sep 17 00:00:00 2001 From: Gaurav Saini Date: Wed, 27 Mar 2024 16:17:20 +0000 Subject: [PATCH 18/21] changes to test files for terragrunt & cosign --- test/terraform/terragrunt_fallback_test.sh | 3 ++- test/terraform/tflint_fallback_test.sh | 15 ++------------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/test/terraform/terragrunt_fallback_test.sh b/test/terraform/terragrunt_fallback_test.sh index 510b54d5f..8eeb649a2 100644 --- a/test/terraform/terragrunt_fallback_test.sh +++ b/test/terraform/terragrunt_fallback_test.sh @@ -183,7 +183,8 @@ try_install_dummy_terragrunt_version() { echo "Downloading Terragrunt... v${TERRAGRUNT_VERSION}" terragrunt_filename="terragrunt_linux_${architecture}" install_terragrunt "$TERRAGRUNT_VERSION" - if grep -q "Not Found" "/tmp/tf-downloads/${terragrunt_filename}"; then + output=$(cat "/tmp/tf-downloads/${terragrunt_filename}") + if [[ $output == "Not Found" ]]; then install_previous_version TERRAGRUNT_VERSION $terragrunt_url $mode fi if [ "${TERRAGRUNT_SHA256}" != "dev-mode" ]; then diff --git a/test/terraform/tflint_fallback_test.sh b/test/terraform/tflint_fallback_test.sh index 6d43d2259..45f89f788 100644 --- a/test/terraform/tflint_fallback_test.sh +++ b/test/terraform/tflint_fallback_test.sh @@ -171,23 +171,14 @@ install_previous_version() { echo "${given_version}=${!given_version}" } -# Function to check if URL returns 404 -check_failure() { - local url="$1" - local resp_code=$2 - local response_code=$(curl -o /dev/null -s -w "%{http_code}\n" "$url") - declare -g ${resp_code}="$response_code" -} - install_cosign() { COSIGN_VERSION=$1 local URL=$2 local mode=$3 cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" - resp_code=200 - check_failure "$cosign_url" resp_code - if [ "$resp_code" -eq 404 ] || [ "$resp_code" -eq 302 ]; then + curl -L "${cosign_url}" -o $cosign_filename + if grep -q "Not Found" "$cosign_filename"; then echo -e "\n(!) Failed to fetch the latest artifacts for cosign v${COSIGN_VERSION}..." REPO_URL=$(get_github_api_repo_url "$URL") get_previous_version "$URL" "$REPO_URL" COSIGN_VERSION $mode @@ -195,8 +186,6 @@ install_cosign() { cosign_filename="/tmp/cosign_${COSIGN_VERSION}_${architecture}.deb" cosign_url="https://github.com/sigstore/cosign/releases/latest/download/cosign_${COSIGN_VERSION}_${architecture}.deb" curl -L "${cosign_url}" -o $cosign_filename - else - curl -L "${cosign_url}" -o $cosign_filename fi dpkg -i $cosign_filename rm $cosign_filename From 9c7be43d40ceb2ded2fcf51e4ff6adfdd5ebb0be Mon Sep 17 00:00:00 2001 From: Gaurav Saini Date: Wed, 27 Mar 2024 22:30:37 +0000 Subject: [PATCH 19/21] Changes acc. to review comments for pr --- src/terraform/install.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/terraform/install.sh b/src/terraform/install.sh index db067e98d..253759517 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -226,12 +226,12 @@ get_previous_version() { prev_version=${!variable_name} output=$(curl -s "$repo_url"); - # checking if jq package exists - if ! command -v jq &> /dev/null - then - echo "jq could not be found, attempting to install..." - apt-get update && apt-get install -y jq + + # install jq if not exists + if ! type jq > /dev/null 2>&1; then + check_packages jq fi + message=$(echo "$output" | jq -r '.message') if [[ $message == "API rate limit exceeded"* ]]; then @@ -421,6 +421,7 @@ install_terragrunt() { if [ "${TERRAGRUNT_VERSION}" != "none" ]; then echo "Downloading Terragrunt..." + TERRAGRUNT_VERSION="0.55.XYZ" terragrunt_filename="terragrunt_linux_${architecture}" install_terragrunt "$TERRAGRUNT_VERSION" output=$(cat "/tmp/tf-downloads/${terragrunt_filename}") From 0dc47b864b6999f5fdc421fcb9a4dbdc0afb7f30 Mon Sep 17 00:00:00 2001 From: Gaurav Saini Date: Wed, 27 Mar 2024 22:35:02 +0000 Subject: [PATCH 20/21] changes requested --- .../terraform/terraform_docs_fallback_test.sh | 26 +++++++++++++++---- test/terraform/terraform_fallback_test.sh | 24 +++++++++++++---- test/terraform/terragrunt_fallback_test.sh | 25 ++++++++++++++---- test/terraform/tflint_fallback_test.sh | 24 +++++++++++++---- test/terraform/tfsec_fallback_test.sh | 24 +++++++++++++---- 5 files changed, 98 insertions(+), 25 deletions(-) diff --git a/test/terraform/terraform_docs_fallback_test.sh b/test/terraform/terraform_docs_fallback_test.sh index ab7d03b43..bcd63b5eb 100644 --- a/test/terraform/terraform_docs_fallback_test.sh +++ b/test/terraform/terraform_docs_fallback_test.sh @@ -104,6 +104,22 @@ find_prev_version_from_git_tags() { set -e } +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + # Function to fetch the version released prior to the latest version get_previous_version() { local url=$1 @@ -113,12 +129,12 @@ get_previous_version() { prev_version=${!variable_name} output=$(curl -s "$repo_url"); - # checking if jq package exists - if ! command -v jq &> /dev/null - then - echo "jq could not be found, attempting to install..." - apt-get update && apt-get install -y jq + + # install jq if not exists + if ! type jq > /dev/null 2>&1; then + check_packages jq fi + message=$(echo "$output" | jq -r '.message') if [[ "$mode" == "mode1" ]]; then message="API rate limit exceeded"; diff --git a/test/terraform/terraform_fallback_test.sh b/test/terraform/terraform_fallback_test.sh index f9d34f9cf..00ba3dc0b 100644 --- a/test/terraform/terraform_fallback_test.sh +++ b/test/terraform/terraform_fallback_test.sh @@ -101,6 +101,22 @@ find_prev_version_from_git_tags() { set -e } +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + # Function to fetch the version released prior to the latest version get_previous_version() { local url=$1 @@ -110,11 +126,9 @@ get_previous_version() { prev_version=${!variable_name} output=$(curl -s "$repo_url"); - # checking if jq package exists - if ! command -v jq &> /dev/null - then - echo "jq could not be found, attempting to install..." - apt-get update && apt-get install -y jq + # install jq if not exists + if ! type jq > /dev/null 2>&1; then + check_packages jq fi message=$(echo "$output" | jq -r '.message') if [[ "$mode" == "mode1" ]]; then diff --git a/test/terraform/terragrunt_fallback_test.sh b/test/terraform/terragrunt_fallback_test.sh index 8eeb649a2..5a1207a72 100644 --- a/test/terraform/terragrunt_fallback_test.sh +++ b/test/terraform/terragrunt_fallback_test.sh @@ -103,6 +103,23 @@ find_prev_version_from_git_tags() { set -e } +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + + # Function to fetch the version released prior to the latest version get_previous_version() { local url=$1 @@ -112,11 +129,9 @@ get_previous_version() { prev_version=${!variable_name} output=$(curl -s "$repo_url"); - # checking if jq package exists - if ! command -v jq &> /dev/null - then - echo "jq could not be found, attempting to install..." - apt-get update && apt-get install -y jq + # install jq if not exists + if ! type jq > /dev/null 2>&1; then + check_packages jq fi message=$(echo "$output" | jq -r '.message') if [[ "$mode" == "mode1" ]]; then diff --git a/test/terraform/tflint_fallback_test.sh b/test/terraform/tflint_fallback_test.sh index 45f89f788..c4a10269f 100644 --- a/test/terraform/tflint_fallback_test.sh +++ b/test/terraform/tflint_fallback_test.sh @@ -108,6 +108,22 @@ find_prev_version_from_git_tags() { set -e } +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + # Function to fetch the version released prior to the latest version get_previous_version() { local url=$1 @@ -117,11 +133,9 @@ get_previous_version() { prev_version=${!variable_name} output=$(curl -s "$repo_url"); - # checking if jq package exists - if ! command -v jq &> /dev/null - then - echo "jq could not be found, attempting to install..." - apt-get update && apt-get install -y jq + # install jq if not exists + if ! type jq > /dev/null 2>&1; then + check_packages jq fi message=$(echo "$output" | jq -r '.message') if [[ "$mode" == "mode1" ]]; then diff --git a/test/terraform/tfsec_fallback_test.sh b/test/terraform/tfsec_fallback_test.sh index 63d6362e1..f59540b8a 100644 --- a/test/terraform/tfsec_fallback_test.sh +++ b/test/terraform/tfsec_fallback_test.sh @@ -107,6 +107,22 @@ find_prev_version_from_git_tags() { set -e } +apt_get_update() +{ + if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then + echo "Running apt-get update..." + apt-get update -y + fi +} + +# Checks if packages are installed and installs them if not +check_packages() { + if ! dpkg -s "$@" > /dev/null 2>&1; then + apt_get_update + apt-get -y install --no-install-recommends "$@" + fi +} + # Function to fetch the version released prior to the latest version get_previous_version() { local url=$1 @@ -116,11 +132,9 @@ get_previous_version() { prev_version=${!variable_name} output=$(curl -s "$repo_url"); - # checking if jq package exists - if ! command -v jq &> /dev/null - then - echo "jq could not be found, attempting to install..." - apt-get update && apt-get install -y jq + # install jq if not exists + if ! type jq > /dev/null 2>&1; then + check_packages jq fi message=$(echo "$output" | jq -r '.message') if [[ "$mode" == "mode1" ]]; then From 0f3bfaf312cbaa4589622f9559f6bce4405d5002 Mon Sep 17 00:00:00 2001 From: Gaurav Saini Date: Wed, 27 Mar 2024 23:40:29 +0000 Subject: [PATCH 21/21] changes as requested in review comments ! --- src/terraform/install.sh | 27 ++++++------------- .../terraform/terraform_docs_fallback_test.sh | 18 +++---------- test/terraform/terraform_fallback_test.sh | 20 +++++--------- test/terraform/terragrunt_fallback_test.sh | 20 +++++--------- test/terraform/tflint_fallback_test.sh | 20 +++++--------- test/terraform/tfsec_fallback_test.sh | 22 ++++++--------- 6 files changed, 38 insertions(+), 89 deletions(-) diff --git a/src/terraform/install.sh b/src/terraform/install.sh index 253759517..dc7aef8c3 100755 --- a/src/terraform/install.sh +++ b/src/terraform/install.sh @@ -227,10 +227,8 @@ get_previous_version() { output=$(curl -s "$repo_url"); - # install jq if not exists - if ! type jq > /dev/null 2>&1; then - check_packages jq - fi + # install jq + check_packages jq message=$(echo "$output" | jq -r '.message') @@ -252,25 +250,17 @@ get_github_api_repo_url() { echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } -get_pkg_name() { - local input_string="$1" - local lowercase_input="${input_string,,}" # Convert to lowercase - local suffix="_version" - local substring="${lowercase_input%$suffix*}" # Remove suffix and everything after it - echo "$substring" -} - install_previous_version() { given_version=$1 requested_version=${!given_version} local URL=$2 + INSTALLER_FN=$3 local REPO_URL=$(get_github_api_repo_url "$URL") local PKG_NAME=$(get_pkg_name "${given_version}") echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." get_previous_version "$URL" "$REPO_URL" requested_version echo -e "\nAttempting to install ${requested_version}" declare -g ${given_version}="${requested_version#v}" - INSTALLER_FN="install_${PKG_NAME}" $INSTALLER_FN "${!given_version}" echo "${given_version}=${!given_version}" } @@ -344,7 +334,7 @@ echo "Downloading terraform..." terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" install_terraform "$TERRAFORM_VERSION" if grep -q "The specified key does not exist." "${terraform_filename}"; then - install_previous_version TERRAFORM_VERSION $terraform_url + install_previous_version TERRAFORM_VERSION $terraform_url "install_terraform" terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" fi if [ "${TERRAFORM_SHA256}" != "dev-mode" ]; then @@ -371,7 +361,7 @@ if [ "${TFLINT_VERSION}" != "none" ]; then TFLINT_FILENAME="tflint_linux_${architecture}.zip" install_tflint "$TFLINT_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${TFLINT_FILENAME}"; then - install_previous_version TFLINT_VERSION "$tflint_url" + install_previous_version TFLINT_VERSION "$tflint_url" "install_tflint" fi if [ "${TFLINT_SHA256}" != "dev-mode" ]; then @@ -421,12 +411,11 @@ install_terragrunt() { if [ "${TERRAGRUNT_VERSION}" != "none" ]; then echo "Downloading Terragrunt..." - TERRAGRUNT_VERSION="0.55.XYZ" terragrunt_filename="terragrunt_linux_${architecture}" install_terragrunt "$TERRAGRUNT_VERSION" output=$(cat "/tmp/tf-downloads/${terragrunt_filename}") if [[ $output == "Not Found" ]]; then - install_previous_version TERRAGRUNT_VERSION $terragrunt_url + install_previous_version TERRAGRUNT_VERSION $terragrunt_url "install_terragrunt" fi if [ "${TERRAGRUNT_SHA256}" != "dev-mode" ]; then if [ "${TERRAGRUNT_SHA256}" = "automatic" ]; then @@ -479,7 +468,7 @@ if [ "${INSTALL_TFSEC}" = "true" ]; then echo "(*) Downloading TFSec... ${tfsec_filename}" install_tfsec "$TFSEC_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${tfsec_filename}"; then - install_previous_version TFSEC_VERSION $tfsec_url + install_previous_version TFSEC_VERSION $tfsec_url "install_tfsec" tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" fi if [ "${TFSEC_SHA256}" != "dev-mode" ]; then @@ -510,7 +499,7 @@ if [ "${INSTALL_TERRAFORM_DOCS}" = "true" ]; then echo "(*) Downloading Terraform docs... ${tfdocs_filename}" install_terraform_docs "$TERRAFORM_DOCS_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${tfdocs_filename}"; then - install_previous_version TERRAFORM_DOCS_VERSION $terraform_docs_url + install_previous_version TERRAFORM_DOCS_VERSION $terraform_docs_url "install_terraform_docs" tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" fi if [ "${TERRAFORM_DOCS_SHA256}" != "dev-mode" ]; then diff --git a/test/terraform/terraform_docs_fallback_test.sh b/test/terraform/terraform_docs_fallback_test.sh index bcd63b5eb..256ef2f85 100644 --- a/test/terraform/terraform_docs_fallback_test.sh +++ b/test/terraform/terraform_docs_fallback_test.sh @@ -130,10 +130,8 @@ get_previous_version() { output=$(curl -s "$repo_url"); - # install jq if not exists - if ! type jq > /dev/null 2>&1; then - check_packages jq - fi + # install jq + check_packages jq message=$(echo "$output" | jq -r '.message') if [[ "$mode" == "mode1" ]]; then @@ -159,26 +157,18 @@ get_github_api_repo_url() { echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } -get_pkg_name() { - local input_string="$1" - local lowercase_input="${input_string,,}" # Convert to lowercase - local suffix="_version" - local substring="${lowercase_input%$suffix*}" # Remove suffix and everything after it - echo "$substring" -} - install_previous_version() { given_version=$1 requested_version=${!given_version} local URL=$2 local mode=$3 + INSTALLER_FN=$4 local REPO_URL=$(get_github_api_repo_url "$URL") local PKG_NAME=$(get_pkg_name "${given_version}") echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." get_previous_version "$URL" "$REPO_URL" requested_version $mode echo -e "\nAttempting to install ${requested_version}" declare -g ${given_version}="${requested_version#v}" - INSTALLER_FN="install_${PKG_NAME}" $INSTALLER_FN "${!given_version}" echo "${given_version}=${!given_version}" } @@ -201,7 +191,7 @@ try_install_terraform_docs_dummy_version() { echo "(*) Downloading Terraform docs... ${tfdocs_filename}" install_terraform_docs "$TERRAFORM_DOCS_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${tfdocs_filename}"; then - install_previous_version TERRAFORM_DOCS_VERSION $terraform_docs_url $mode + install_previous_version TERRAFORM_DOCS_VERSION $terraform_docs_url $mode "install_terraform_docs" tfdocs_filename="terraform-docs-v${TERRAFORM_DOCS_VERSION}-linux-${architecture}.tar.gz" fi if [ "${TERRAFORM_DOCS_SHA256}" != "dev-mode" ]; then diff --git a/test/terraform/terraform_fallback_test.sh b/test/terraform/terraform_fallback_test.sh index 00ba3dc0b..a5193daa4 100644 --- a/test/terraform/terraform_fallback_test.sh +++ b/test/terraform/terraform_fallback_test.sh @@ -126,10 +126,10 @@ get_previous_version() { prev_version=${!variable_name} output=$(curl -s "$repo_url"); - # install jq if not exists - if ! type jq > /dev/null 2>&1; then - check_packages jq - fi + + # install jq + check_packages jq + message=$(echo "$output" | jq -r '.message') if [[ "$mode" == "mode1" ]]; then message="API rate limit exceeded"; @@ -154,26 +154,18 @@ get_github_api_repo_url() { echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } -get_pkg_name() { - local input_string="$1" - local lowercase_input="${input_string,,}" # Convert to lowercase - local suffix="_version" - local substring="${lowercase_input%$suffix*}" # Remove suffix and everything after it - echo "$substring" -} - install_previous_version() { given_version=$1 requested_version=${!given_version} local URL=$2 local mode=$3 + INSTALLER_FN=$4 local REPO_URL=$(get_github_api_repo_url "$URL") local PKG_NAME=$(get_pkg_name "${given_version}") echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." get_previous_version "$URL" "$REPO_URL" requested_version $mode echo -e "\nAttempting to install ${requested_version}" declare -g ${given_version}="${requested_version#v}" - INSTALLER_FN="install_${PKG_NAME}" $INSTALLER_FN "${!given_version}" echo "${given_version}=${!given_version}" } @@ -194,7 +186,7 @@ try_install_dummy_terraform_version() { terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" install_terraform "$TERRAFORM_VERSION" if grep -q "The specified key does not exist." "${terraform_filename}"; then - install_previous_version TERRAFORM_VERSION $terraform_url $mode + install_previous_version TERRAFORM_VERSION $terraform_url $mode "install_terraform" terraform_filename="terraform_${TERRAFORM_VERSION}_linux_${architecture}.zip" fi unzip ${terraform_filename} diff --git a/test/terraform/terragrunt_fallback_test.sh b/test/terraform/terragrunt_fallback_test.sh index 5a1207a72..83289380f 100644 --- a/test/terraform/terragrunt_fallback_test.sh +++ b/test/terraform/terragrunt_fallback_test.sh @@ -129,10 +129,10 @@ get_previous_version() { prev_version=${!variable_name} output=$(curl -s "$repo_url"); - # install jq if not exists - if ! type jq > /dev/null 2>&1; then - check_packages jq - fi + + # install jq + check_packages jq + message=$(echo "$output" | jq -r '.message') if [[ "$mode" == "mode1" ]]; then message="API rate limit exceeded"; @@ -157,26 +157,18 @@ get_github_api_repo_url() { echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } -get_pkg_name() { - local input_string="$1" - local lowercase_input="${input_string,,}" # Convert to lowercase - local suffix="_version" - local substring="${lowercase_input%$suffix*}" # Remove suffix and everything after it - echo "$substring" -} - install_previous_version() { given_version=$1 requested_version=${!given_version} local URL=$2 local mode=$3 + INSTALLER_FN=$4 local REPO_URL=$(get_github_api_repo_url "$URL") local PKG_NAME=$(get_pkg_name "${given_version}") echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." get_previous_version "$URL" "$REPO_URL" requested_version $mode echo -e "\nAttempting to install ${requested_version}" declare -g ${given_version}="${requested_version#v}" - INSTALLER_FN="install_${PKG_NAME}" $INSTALLER_FN "${!given_version}" echo "${given_version}=${!given_version}" } @@ -200,7 +192,7 @@ try_install_dummy_terragrunt_version() { install_terragrunt "$TERRAGRUNT_VERSION" output=$(cat "/tmp/tf-downloads/${terragrunt_filename}") if [[ $output == "Not Found" ]]; then - install_previous_version TERRAGRUNT_VERSION $terragrunt_url $mode + install_previous_version TERRAGRUNT_VERSION $terragrunt_url $mode "install_terragrunt" fi if [ "${TERRAGRUNT_SHA256}" != "dev-mode" ]; then if [ "${TERRAGRUNT_SHA256}" = "automatic" ]; then diff --git a/test/terraform/tflint_fallback_test.sh b/test/terraform/tflint_fallback_test.sh index c4a10269f..5619ff4fb 100644 --- a/test/terraform/tflint_fallback_test.sh +++ b/test/terraform/tflint_fallback_test.sh @@ -133,10 +133,10 @@ get_previous_version() { prev_version=${!variable_name} output=$(curl -s "$repo_url"); - # install jq if not exists - if ! type jq > /dev/null 2>&1; then - check_packages jq - fi + + # install jq + check_packages jq + message=$(echo "$output" | jq -r '.message') if [[ "$mode" == "mode1" ]]; then message="API rate limit exceeded"; @@ -161,26 +161,18 @@ get_github_api_repo_url() { echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } -get_pkg_name() { - local input_string="$1" - local lowercase_input="${input_string,,}" # Convert to lowercase - local suffix="_version" - local substring="${lowercase_input%$suffix*}" # Remove suffix and everything after it - echo "$substring" -} - install_previous_version() { given_version=$1 requested_version=${!given_version} local URL=$2 local mode=$3 + INSTALLER_FN=$4 local REPO_URL=$(get_github_api_repo_url "$URL") local PKG_NAME=$(get_pkg_name "${given_version}") echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." get_previous_version "$URL" "$REPO_URL" requested_version $mode echo -e "\nAttempting to install ${requested_version}" declare -g ${given_version}="${requested_version#v}" - INSTALLER_FN="install_${PKG_NAME}" $INSTALLER_FN "${!given_version}" echo "${given_version}=${!given_version}" } @@ -241,7 +233,7 @@ try_install_dummy_tflint_cosign_version() { TFLINT_FILENAME="tflint_linux_${architecture}.zip" install_tflint "$TFLINT_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${TFLINT_FILENAME}"; then - install_previous_version TFLINT_VERSION "$tflint_url" $mode + install_previous_version TFLINT_VERSION "$tflint_url" $mode "install_tflint" fi if [ "${TFLINT_SHA256}" != "dev-mode" ]; then diff --git a/test/terraform/tfsec_fallback_test.sh b/test/terraform/tfsec_fallback_test.sh index f59540b8a..5c87d821e 100644 --- a/test/terraform/tfsec_fallback_test.sh +++ b/test/terraform/tfsec_fallback_test.sh @@ -132,16 +132,18 @@ get_previous_version() { prev_version=${!variable_name} output=$(curl -s "$repo_url"); - # install jq if not exists - if ! type jq > /dev/null 2>&1; then - check_packages jq - fi + + # install jq + check_packages jq + message=$(echo "$output" | jq -r '.message') + if [[ "$mode" == "mode1" ]]; then message="API rate limit exceeded"; elif [[ "$mode" == "mode2" ]]; then message="" fi + if [[ $message == "API rate limit exceeded"* ]]; then echo -e "\nAn attempt to find latest version using GitHub Api Failed... \nReason: ${message}" echo -e "\nAttempting to find latest version using GitHub tags." @@ -160,26 +162,18 @@ get_github_api_repo_url() { echo "${url/https:\/\/github.com/https:\/\/api.github.com\/repos}/releases/latest" } -get_pkg_name() { - local input_string="$1" - local lowercase_input="${input_string,,}" # Convert to lowercase - local suffix="_version" - local substring="${lowercase_input%$suffix*}" # Remove suffix and everything after it - echo "$substring" -} - install_previous_version() { given_version=$1 requested_version=${!given_version} local URL=$2 local mode=$3 + INSTALLER_FN=$4 local REPO_URL=$(get_github_api_repo_url "$URL") local PKG_NAME=$(get_pkg_name "${given_version}") echo -e "\n(!) Failed to fetch the latest artifacts for ${PKG_NAME} v${requested_version}..." get_previous_version "$URL" "$REPO_URL" requested_version $mode echo -e "\nAttempting to install ${requested_version}" declare -g ${given_version}="${requested_version#v}" - INSTALLER_FN="install_${PKG_NAME}" $INSTALLER_FN "${!given_version}" echo "${given_version}=${!given_version}" } @@ -201,7 +195,7 @@ try_install_tfsec_dummy_version() { echo "(*) Downloading TFSec... ${tfsec_filename}" install_tfsec "$TFSEC_VERSION" if grep -q "Not Found" "/tmp/tf-downloads/${tfsec_filename}"; then - install_previous_version TFSEC_VERSION $tfsec_url $mode + install_previous_version TFSEC_VERSION $tfsec_url $mode "install_tfsec" tfsec_filename="tfsec_${TFSEC_VERSION}_linux_${architecture}.tar.gz" fi if [ "${TFSEC_SHA256}" != "dev-mode" ]; then