From 6935171aefeaabeec449b05a51471e93fe78a439 Mon Sep 17 00:00:00 2001 From: Chuck Lantz Date: Sat, 5 Nov 2022 04:18:21 +0000 Subject: [PATCH 1/4] Fix #258, improve tests, fix oryx bug --- .devcontainer.json | 5 ++- src/python/devcontainer-feature.json | 5 +-- src/python/install.sh | 17 ++++++--- test/python/install_additional_jupyterlab.sh | 0 test/python/install_additional_python.sh | 33 ++++++++++++++--- test/python/install_jupyterlab.sh | 0 test/python/install_os_provided_python.sh | 23 +++++++++++- test/python/install_via_oryx.sh | 37 ++++++++++++++++++++ test/python/scenarios.json | 7 +++- test/python/test.sh | 25 +++++++++++++ 10 files changed, 137 insertions(+), 15 deletions(-) mode change 100644 => 100755 test/python/install_additional_jupyterlab.sh mode change 100644 => 100755 test/python/install_additional_python.sh mode change 100644 => 100755 test/python/install_jupyterlab.sh mode change 100644 => 100755 test/python/install_os_provided_python.sh create mode 100755 test/python/install_via_oryx.sh diff --git a/.devcontainer.json b/.devcontainer.json index d5d850ca5..81b3c1ff6 100644 --- a/.devcontainer.json +++ b/.devcontainer.json @@ -11,5 +11,8 @@ "features": { "ghcr.io/devcontainers/features/docker-in-docker:1": {} }, - "postCreateCommand": "npm install -g @devcontainers/cli" + "postCreateCommand": "npm install -g @devcontainers/cli", + "hostRequirements": { + "cpus": 4 + } } diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index f2d116e2a..aa326660f 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -60,7 +60,7 @@ "ms-python.vscode-pylance" ], "settings": { - "python.defaultInterpreterPath": "/usr/local/bin/python", + "python.defaultInterpreterPath": "/usr/local/python/current/bin/python", "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", "python.formatting.blackPath": "/usr/local/py-utils/bin/black", "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", @@ -74,6 +74,7 @@ } }, "installsAfter": [ - "ghcr.io/devcontainers/features/common-utils" + "ghcr.io/devcontainers/features/common-utils", + "ghcr.io/devcontainers/features/oryx" ] } diff --git a/src/python/install.sh b/src/python/install.sh index ac5eec6a2..b7b306136 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -299,13 +299,16 @@ install_using_oryx() { echo "(!) Python version ${VERSION} already exists." exit 1 fi + + # The python install root path may not exist, so create it + mkdir -p "${PYTHON_INSTALL_PATH}" oryx_install "python" "${VERSION}" "${INSTALL_PATH}" "lib" || return 1 ln -s "${INSTALL_PATH}/bin/idle3" "${INSTALL_PATH}/bin/idle" ln -s "${INSTALL_PATH}/bin/pydoc3" "${INSTALL_PATH}/bin/pydoc" ln -s "${INSTALL_PATH}/bin/python3-config" "${INSTALL_PATH}/bin/python-config" - add_symlink + add_symlink } sudo_if() { @@ -339,11 +342,15 @@ install_python() { # If the os-provided versions are "good enough", detect that and bail out. if [ ${PYTHON_VERSION} = "os-provided" ] || [ ${PYTHON_VERSION} = "system" ]; then check_packages python3 python3-doc python3-pip python3-venv python3-dev python3-tk - PYTHON_ROOT="/usr/bin" + INSTALL_PATH="/usr" + + ln -s "${INSTALL_PATH}/bin/python3" "${INSTALL_PATH}/bin/python" + ln -s "${INSTALL_PATH}/bin/pydoc3" "${INSTALL_PATH}/bin/pydoc" + ln -s "${INSTALL_PATH}/bin/python3-config" "${INSTALL_PATH}/bin/python-config" - ln -s "${PYTHON_ROOT}/python3" "${PYTHON_ROOT}/python" - ln -s "${PYTHON_ROOT}/pydoc3" "${PYTHON_ROOT}/pydoc" - ln -s "${PYTHON_ROOT}/python3-config" "${PYTHON_ROOT}/python-config" + # Add the current symlink but point it to "/usr" since python is at /usr/bin/python + mkdir -p "${PYTHON_INSTALL_PATH}" + add_symlink should_install_from_source=false elif [ "$(dpkg --print-architecture)" = "amd64" ] && [ "${USE_ORYX_IF_AVAILABLE}" = "true" ] && type oryx > /dev/null 2>&1; then diff --git a/test/python/install_additional_jupyterlab.sh b/test/python/install_additional_jupyterlab.sh old mode 100644 new mode 100755 diff --git a/test/python/install_additional_python.sh b/test/python/install_additional_python.sh old mode 100644 new mode 100755 index 00e43d81c..bbd6c137a --- a/test/python/install_additional_python.sh +++ b/test/python/install_additional_python.sh @@ -5,11 +5,34 @@ set -e # Optional: Import test library source dev-container-features-test-lib -check "python version 3.11 installed as default" python --version | grep 3.11 -check "python3 version 3.11 installed as default" python3 --version | grep 3.11 -check "python version 3.10.5 installed" ls -l /usr/local/python | grep 3.10.5 -check "python version 3.8.13 installed" ls -l /usr/local/python | grep 3.8.13 -check "python version 3.9.13 installed" ls -l /usr/local/python | grep 3.9.13 +check "python version 3.11 installed as default" bash -c "python --version | grep 3.11" +check "python3 version 3.11 installed as default" bash -c "python3 --version | grep 3.11" +check "python version 3.10.5 installed" bash -c "ls -l /usr/local/python | grep 3.10.5" +check "python version 3.8 installed" bash -c "ls -l /usr/local/python | grep 3.8" +check "python version 3.9.13 installed" bash -c "ls -l /usr/local/python | grep 3.9.13" + +# Check that tools can execute - make sure something didn't get messed up in this scenario +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" # Report result reportResults diff --git a/test/python/install_jupyterlab.sh b/test/python/install_jupyterlab.sh old mode 100644 new mode 100755 diff --git a/test/python/install_os_provided_python.sh b/test/python/install_os_provided_python.sh old mode 100644 new mode 100755 index 50a617e93..e65b15484 --- a/test/python/install_os_provided_python.sh +++ b/test/python/install_os_provided_python.sh @@ -10,7 +10,28 @@ check "python is installed" python --version check "pip is installed" pip --version check "pip is installed" pip3 --version -check "node is installed" node --version +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" # Report result reportResults \ No newline at end of file diff --git a/test/python/install_via_oryx.sh b/test/python/install_via_oryx.sh new file mode 100755 index 000000000..e65b15484 --- /dev/null +++ b/test/python/install_via_oryx.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -e + +# Optional: Import test library +source dev-container-features-test-lib + +check "python3 is installed" python3 --version +check "python is installed" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" + +# Report result +reportResults \ No newline at end of file diff --git a/test/python/scenarios.json b/test/python/scenarios.json index 80e0fb5e8..920f4a270 100644 --- a/test/python/scenarios.json +++ b/test/python/scenarios.json @@ -33,8 +33,13 @@ "install_os_provided_python": { "image": "mcr.microsoft.com/devcontainers/base:0-bullseye", "features": { - "node": "latest", "python": "os-provided" } + }, + "install_via_oryx": { + "image": "mcr.microsoft.com/oryx/build:full-debian-bullseye", + "features": { + "python": "3.10" + } } } \ No newline at end of file diff --git a/test/python/test.sh b/test/python/test.sh index 7bd2281c3..8f7d336b3 100755 --- a/test/python/test.sh +++ b/test/python/test.sh @@ -7,6 +7,31 @@ source dev-container-features-test-lib # Definition specific tests check "version" python --version +check "pip is installed" pip --version +check "pip is installed" pip3 --version + +# Check that tools can execute +check "autopep8" autopep8 --version +check "black" black --version +check "yapf" yapf --version +check "bandit" bandit --version +check "flake8" flake8 --version +check "mypy" mypy --version +check "pycodestyle" pycodestyle --version +check "pydocstyle" pydocstyle --version +check "pylint" pylint --version + +# Check paths in settings +check "current symlink is correct" bash -c "which python | grep /usr/local/python/current/bin/python" +check "which autopep8" bash -c "which autopep8 | grep /usr/local/py-utils/bin/autopep8" +check "which black" bash -c "which black | grep /usr/local/py-utils/bin/black" +check "which yapf" bash -c "which yapf | grep /usr/local/py-utils/bin/yapf" +check "which bandit" bash -c "which bandit | grep /usr/local/py-utils/bin/bandit" +check "which flake8" bash -c "which flake8 | grep /usr/local/py-utils/bin/flake8" +check "which mypy" bash -c "which mypy | grep /usr/local/py-utils/bin/mypy" +check "which pycodestyle" bash -c "which pycodestyle | grep /usr/local/py-utils/bin/pycodestyle" +check "which pydocstyle" bash -c "which pydocstyle | grep /usr/local/py-utils/bin/pydocstyle" +check "which pylint" bash -c "which pylint | grep /usr/local/py-utils/bin/pylint" # Report result reportResults From f46db23d24fe91ac4386cfc2f7aa5c02546b13b2 Mon Sep 17 00:00:00 2001 From: Chuck Lantz Date: Mon, 7 Nov 2022 18:00:09 +0000 Subject: [PATCH 2/4] Drop Debian 9 from tests given it is out of support --- .github/workflows/test-all.yaml | 1 - .github/workflows/test-pr.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/test-all.yaml b/.github/workflows/test-all.yaml index 412eb117d..cf876a481 100644 --- a/.github/workflows/test-all.yaml +++ b/.github/workflows/test-all.yaml @@ -47,7 +47,6 @@ jobs: "ubuntu:bionic", "debian:11", "debian:10", - "debian:9", "mcr.microsoft.com/devcontainers/base:ubuntu", "mcr.microsoft.com/devcontainers/base:debian", ] diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index 700c875f9..f89923dc6 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -54,7 +54,6 @@ jobs: "ubuntu:bionic", "debian:11", "debian:10", - "debian:9", "mcr.microsoft.com/devcontainers/base:ubuntu", "mcr.microsoft.com/devcontainers/base:debian", ] From cc560e605dbe2f631a66ec9463e19acb728db4e8 Mon Sep 17 00:00:00 2001 From: Chuck Lantz Date: Mon, 7 Nov 2022 18:04:58 +0000 Subject: [PATCH 3/4] Fix indent --- src/python/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/install.sh b/src/python/install.sh index b7b306136..de1b0e6e3 100755 --- a/src/python/install.sh +++ b/src/python/install.sh @@ -308,7 +308,7 @@ install_using_oryx() { ln -s "${INSTALL_PATH}/bin/pydoc3" "${INSTALL_PATH}/bin/pydoc" ln -s "${INSTALL_PATH}/bin/python3-config" "${INSTALL_PATH}/bin/python-config" - add_symlink + add_symlink } sudo_if() { From 22b066b1c86b444d1a1bc5a450ea31e72ac4480b Mon Sep 17 00:00:00 2001 From: Chuck Lantz Date: Mon, 7 Nov 2022 18:45:56 +0000 Subject: [PATCH 4/4] Bump breakfix version --- src/python/devcontainer-feature.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/devcontainer-feature.json b/src/python/devcontainer-feature.json index aa326660f..8f8faae6a 100644 --- a/src/python/devcontainer-feature.json +++ b/src/python/devcontainer-feature.json @@ -1,6 +1,6 @@ { "id": "python", - "version": "1.0.10", + "version": "1.0.11", "name": "Python", "documentationURL": "https://github.com/devcontainers/features/tree/main/src/python", "description": "Installs the provided version of Python, as well as PIPX, and other common Python utilities. JupyterLab is conditionally installed with the python feature. Note: May require source code compilation.",