Upgrade to Pants 2.10.0rc3 (#7)

Author: Eric-Arellano

3rdparty/python/default_lock.txt

@@ -5,7 +5,7 @@ An example repository to demonstrate codegen support in Pants.
 Refer to these docs for more information:
 
 * [Python Protobuf](https://www.pantsbuild.org/v2.10/docs/protobuf-python)
-* [Python Thrift](https://www.pantsbuild.org/v2.10/docs/protobuf-python)
+* [Python Thrift](https://www.pantsbuild.org/v2.10/docs/thrift-python)
 
 Run `./pants export-codegen ::` to see the generated files. This isn't necessary for Pants to 
 use the generated files, but can be useful when debugging or to generate files for IDEs.

README.md

@@ -12,8 +12,8 @@
 
 set -eou pipefail
 
-# NOTE: To use an unreleased version of Pants from the pantsbuild/pants master branch,
-#  locate the master branch SHA, set PANTS_SHA=<SHA> in the environment, and run this script as usual.
+# NOTE: To use an unreleased version of Pants from the pantsbuild/pants main branch,
+#  locate the main branch SHA, set PANTS_SHA=<SHA> in the environment, and run this script as usual.
 #
 # E.g., PANTS_SHA=725fdaf504237190f6787dda3d72c39010a4c574 ./pants --version
 
@@ -34,9 +34,9 @@ fi
 
 PANTS_BOOTSTRAP="${PANTS_SETUP_CACHE}/bootstrap-$(uname -s)-$(uname -m)"
 
-PEX_VERSION=2.1.42
-PEX_URL="https://github.com/pantsbuild/pex/releases/download/v${PEX_VERSION}/pex"
-PEX_EXPECTED_SHA256="69d6b1b1009b00dd14a3a9f19b72cff818a713ca44b3186c9b12074b2a31e51f"
+_PEX_VERSION=2.1.62
+_PEX_URL="https://github.com/pantsbuild/pex/releases/download/v${_PEX_VERSION}/pex"
+_PEX_EXPECTED_SHA256="56668b1ca147bd63141e586ffee97c7cc51ce8e6eac6c9b7a4bf1215b94396e5"
 
 VIRTUALENV_VERSION=20.4.7
 VIRTUALENV_REQUIREMENTS=$(
@@ -55,6 +55,7 @@ EOF
 
 COLOR_RED="\x1b[31m"
 COLOR_GREEN="\x1b[32m"
+COLOR_YELLOW="\x1b[33m"
 COLOR_RESET="\x1b[0m"
 
 function log() {
@@ -70,6 +71,10 @@ function green() {
   (($# > 0)) && log "${COLOR_GREEN}$*${COLOR_RESET}"
 }
 
+function warn() {
+  (($# > 0)) && log "${COLOR_YELLOW}$*${COLOR_RESET}"
+}
+
 function tempdir {
   mkdir -p "$1"
   mktemp -d "$1"/pants.XXXXXX
@@ -82,13 +87,18 @@ function get_exe_path_or_die {
   fi
 }
 
-function get_pants_config_value {
+function get_pants_config_string_value {
   local config_key="$1"
   local optional_space="[[:space:]]*"
   local prefix="^${config_key}${optional_space}=${optional_space}"
   local raw_value
-  raw_value="$(sed -ne "/${prefix}/ s#${prefix}##p" "${PANTS_TOML}")"
-  echo "${raw_value}"  | tr -d \"\' && return 0
+  raw_value="$(sed -ne "/${prefix}/ s|${prefix}||p" "${PANTS_TOML}")"
+  local optional_suffix="${optional_space}(#.*)?$"
+  echo "${raw_value}" \
+    | sed -E \
+      -e "s|^'([^']*)'${optional_suffix}|\1|" \
+      -e 's|^"([^"]*)"'"${optional_suffix}"'$|\1|' \
+    && return 0
   return 0
 }
 
@@ -120,7 +130,7 @@ function determine_pants_version {
     return
   fi
 
-  pants_version="$(get_pants_config_value 'pants_version')"
+  pants_version="$(get_pants_config_string_value 'pants_version')"
   if [[ -z "${pants_version}" ]]; then
     die "Please explicitly specify the \`pants_version\` in your \`pants.toml\` under the \`[GLOBAL]\` scope.
 See https://pypi.org/project/pantsbuild.pants/#history for all released versions
@@ -168,18 +178,31 @@ function set_supported_python_versions {
   fi
 }
 
+function check_python_exe_compatible_version {
+  local python_exe="$1"
+  local major_minor_version
+  major_minor_version="$(get_python_major_minor_version "${python_exe}")"
+  for valid_version in "${supported_python_versions_int[@]}"; do
+    if [[ "${major_minor_version}" == "${valid_version}" ]]; then
+      echo "${python_exe}" && return 0
+    fi
+  done
+}
+
 function determine_default_python_exe {
-  for version in "${supported_python_versions_decimal[@]}"; do
+  for version in "${supported_python_versions_decimal[@]}" "3" ""; do
     local interpreter_path
     interpreter_path="$(command -v "python${version}")"
     if [[ -z "${interpreter_path}" ]]; then
       continue
     fi
-    # Check if the Python version is installed via Pyenv but not activated.
-    if [[ "$("${interpreter_path}" --version 2>&1 > /dev/null)" == "pyenv: python${version}"* ]]; then
+    # Check if a version is shimmed by pyenv or asdf but not configured.
+    if ! "${interpreter_path}" --version >/dev/null 2>&1; then
       continue
     fi
-    echo "${interpreter_path}" && return 0
+    if [[ -n "$(check_python_exe_compatible_version "${interpreter_path}")" ]]; then
+      echo "${interpreter_path}" && return 0
+    fi
   done
 }
 
@@ -188,25 +211,19 @@ function determine_python_exe {
   set_supported_python_versions "${pants_version}"
   local requirement_str="For \`pants_version = \"${pants_version}\"\`, Pants requires Python ${supported_message} to run."
 
-  local python_bin_name
+  local python_exe
   if [[ "${PYTHON_BIN_NAME}" != 'unspecified' ]]; then
-    python_bin_name="${PYTHON_BIN_NAME}"
+    python_exe="$(get_exe_path_or_die "${PYTHON_BIN_NAME}")" || exit 1
+    if [[ -z "$(check_python_exe_compatible_version "${python_exe}")" ]]; then
+      die "Invalid Python interpreter version for ${python_exe}. ${requirement_str}"
+    fi
   else
-    python_bin_name="$(determine_default_python_exe)"
-    if [[ -z "${python_bin_name}" ]]; then
+    python_exe="$(determine_default_python_exe)"
+    if [[ -z "${python_exe}" ]]; then
       die "No valid Python interpreter found. ${requirement_str} Please check that a valid interpreter is installed and on your \$PATH."
     fi
   fi
-  local python_exe
-  python_exe="$(get_exe_path_or_die "${python_bin_name}")" || exit 1
-  local major_minor_version
-  major_minor_version="$(get_python_major_minor_version "${python_exe}")"
-  for valid_version in "${supported_python_versions_int[@]}"; do
-    if [[ "${major_minor_version}" == "${valid_version}" ]]; then
-      echo "${python_exe}" && return 0
-    fi
-  done
-  die "Invalid Python interpreter version for ${python_exe}. ${requirement_str}"
+  echo "${python_exe}"
 }
 
 function compute_sha256 {
@@ -229,20 +246,25 @@ EOF
 
 function bootstrap_pex {
   local python="$1"
-  local bootstrapped="${PANTS_BOOTSTRAP}/pex-${PEX_VERSION}/pex"
+  local bootstrapped="${PANTS_BOOTSTRAP}/pex-${_PEX_VERSION}/pex"
   if [[ ! -f "${bootstrapped}" ]]; then
     (
       green "Downloading the Pex PEX."
       mkdir -p "${PANTS_BOOTSTRAP}"
       local staging_dir
       staging_dir=$(tempdir "${PANTS_BOOTSTRAP}")
       cd "${staging_dir}"
-      curl -LO "${PEX_URL}"
+      curl --proto "=https" \
+           --tlsv1.2 \
+           --silent \
+           --location \
+           --remote-name \
+           "${_PEX_URL}"
       fingerprint="$(compute_sha256 "${python}" "pex")"
-      if [[ "${PEX_EXPECTED_SHA256}" != "${fingerprint}" ]]; then
-        die "SHA256 of ${PEX_URL} is not as expected. Aborting."
+      if [[ "${_PEX_EXPECTED_SHA256}" != "${fingerprint}" ]]; then
+        die "SHA256 of ${_PEX_URL} is not as expected. Aborting."
       fi
-      green "SHA256 fingerprint of ${PEX_URL} verified."
+      green "SHA256 fingerprint of ${_PEX_URL} verified."
       mkdir -p "$(dirname "${bootstrapped}")"
       mv -f "${staging_dir}/pex" "${bootstrapped}"
       rmdir "${staging_dir}"
@@ -251,6 +273,15 @@ function bootstrap_pex {
   echo "${bootstrapped}"
 }
 
+function scrub_PEX_env_vars {
+  # Ensure the virtualenv PEX runs as shrink-wrapped.
+  # See: https://github.com/pantsbuild/setup/issues/105
+  if [[ -n "${!PEX_@}" ]]; then
+    warn "Scrubbing ${!PEX_@}"
+    unset "${!PEX_@}"
+  fi
+}
+
 function bootstrap_virtualenv {
   local python="$1"
   local bootstrapped="${PANTS_BOOTSTRAP}/virtualenv-${VIRTUALENV_VERSION}/virtualenv.pex"
@@ -263,7 +294,10 @@ function bootstrap_virtualenv {
       staging_dir=$(tempdir "${PANTS_BOOTSTRAP}")
       cd "${staging_dir}"
       echo "${VIRTUALENV_REQUIREMENTS}" > requirements.txt
-      "${python}" "${pex_path}" -r requirements.txt -c virtualenv -o virtualenv.pex
+      (
+        scrub_PEX_env_vars
+        "${python}" "${pex_path}" -r requirements.txt -c virtualenv -o virtualenv.pex
+      )
       mkdir -p "$(dirname "${bootstrapped}")"
       mv -f "${staging_dir}/virtualenv.pex" "${bootstrapped}"
       rm -rf "${staging_dir}"
@@ -283,7 +317,12 @@ function get_version_for_sha {
 
   # Retrieve the Pants version associated with this commit.
   local pants_version
-  pants_version="$(curl --fail -sL "https://raw.githubusercontent.com/pantsbuild/pants/${sha}/src/python/pants/VERSION")"
+  pants_version="$(curl --proto "=https" \
+                        --tlsv1.2 \
+                        --fail \
+                        --silent \
+                        --location \
+                        "https://raw.githubusercontent.com/pantsbuild/pants/${sha}/src/python/pants/VERSION")"
 
   # Construct the version as the release version from src/python/pants/VERSION, plus the string `+gitXXXXXXXX`,
   # where the XXXXXXXX is the first 8 characters of the SHA.
@@ -315,10 +354,15 @@ function bootstrap_pants {
       local virtualenv_path
       virtualenv_path="$(bootstrap_virtualenv "${python}")" || exit 1
       green "Installing ${pants_requirement} into a virtual environment at ${bootstrapped}"
-      # shellcheck disable=SC2086
-      "${python}" "${virtualenv_path}" --no-download "${staging_dir}/install" && \
-      "${staging_dir}/install/bin/pip" install -U pip && \
-      "${staging_dir}/install/bin/pip" install ${maybe_find_links} --progress-bar off "${pants_requirement}" && \
+      (
+        scrub_PEX_env_vars
+        # shellcheck disable=SC2086
+        "${python}" "${virtualenv_path}" --quiet --no-download "${staging_dir}/install" && \
+        # Grab the latest pip, but don't advance setuptools past 58 which drops support for the
+        # `setup` kwarg `use_2to3` which Pants 1.x sdist dependencies (pystache) use.
+        "${staging_dir}/install/bin/pip" install --quiet -U pip "setuptools<58" && \
+        "${staging_dir}/install/bin/pip" install ${maybe_find_links} --quiet --progress-bar off "${pants_requirement}"
+      ) && \
       ln -s "${staging_dir}/install" "${staging_dir}/${target_folder_name}" && \
       mv "${staging_dir}/${target_folder_name}" "${bootstrapped}" && \
       green "New virtual environment successfully created at ${bootstrapped}."

pants

@@ -1,5 +1,5 @@
 [GLOBAL]
-pants_version = "2.10.0rc0"
+pants_version = "2.10.0rc3"
 backend_packages = [
   "pants.backend.codegen.protobuf.python",
   "pants.backend.codegen.thrift.apache.python",
@@ -14,10 +14,3 @@ repo_id = "205E301D-44A4-42FF-8495-C9D41871A431"
 
 [python]
 interpreter_constraints = ["CPython>=3.8,<3.11"]
-enable_resolves = true
-
-[python-protobuf]
-runtime_dependencies = ["3rdparty/python#protobuf"]
-
-[python-thrift]
-runtime_dependencies = ["3rdparty/python#thrift"]