Remove Python tarballs from pythonSrc/ext dir

LICENSE

@@ -218,11 +218,7 @@ The PostgreSQL software includes:
       src/bin/pg_controldata/pg_controldata.c
       copyright (c) Oliver Elphick <olly@lfix.co.uk>, 2001
 
-      gpMgmt/bin/pythonSrc/ext/behave-1.2.6.tar.gz
-      see licenses/LICENSE-bsd2-behave.txt
 
-      gpMgmt/bin/pythonSrc/ext/mock-1.0.1.tar.gz
-      see licenses/LICENSE-bsd2-mock.txt
 
 ----------------------------
    BSD-4-Clause (University of California-Specific)
@@ -250,9 +246,6 @@ The PostgreSQL software includes:
       src/backend/libpq/sha2.h
       see licenses/LICENSE-bsd3-sha2.txt
 
-      gpMgmt/bin/pythonSrc/ext/psutil-5.7.0.tar.gz
-      see licenses/LICENSE-bsd3-psutil.txt
-
       contrib/pax_storage/src/cpp/contrib/googletest
       see licenses/LICENSE-googletest.txt
 
@@ -262,15 +255,6 @@ The PostgreSQL software includes:
 ----------------------------
     MIT LICENSE
 
-      gpMgmt/bin/pythonSrc/ext/PyYAML-5.4.1.tar.gz
-      see licenses/LICENSE-pyyaml.txt
-
-      gpMgmt/bin/pythonSrc/ext/setuptools-36.6.0.tar.gz
-      see licenses/LICENSE-setuptools.txt
-
-      gpMgmt/bin/pythonSrc/ext/parse-1.8.2.tar.gz
-      see licenses/LICENSE-parse.txt
-
       dependencies/yyjson
       see licenses/LICENSE-yyjson.txt
 
@@ -312,21 +296,18 @@ The PostgreSQL software includes:
    test-ctype LICENSE
 
       src/test/locale/test-ctype.c
-
       see licenses/LICENSE-test-ctype.txt
 
 ----------------------------
    regex LICENSE
 
       src/backend/regex 
-
       see licenses/LICENSE-regex.txt
 
 ----------------------------
    PostgreSQL-style License
 
       src/port/gettimeofday.c 
-
       see licenses/LICENSE-gettimeofday.txt
 
 ================================================================================
@@ -349,22 +330,14 @@ The Greenplum Database software includes:
    Stream License
 
       gpMgmt/bin/stream/*
-
       see licenses/LICENSE-stream.txt
 
 ----------------------------
    0-clause license ("Zero Clause BSD")
 
       gpcontrib/orafce/*
-
       see licenses/LICENSE-orafce.txt
 
-================================================================================
-PostgreSQL-style license
-
-   gpMgmt/bin/pythonSrc/ext/PyGreSQL-5.2.tar.gz
-   see licenses/LICENSE-pygresql.txt
-
 ================================================================================
 This product includes code from pg_cron under PostgreSQL license:
 

gpAux/client/install/src/windows/CreatePackage.bat

@@ -11,8 +11,8 @@ type nul > %GPDB_INSTALL_PATH%\bin\gppylib\__init__.py
 copy ..\..\..\..\..\gpMgmt\bin\gppylib\gpversion.py %GPDB_INSTALL_PATH%\bin\gppylib\
 perl -pi.bak -e "s,\$Revision\$,%VERSION%," %GPDB_INSTALL_PATH%\bin\gpload.py
 copy ..\..\..\..\..\gpMgmt\bin\gpload.bat %GPDB_INSTALL_PATH%\bin
-for %%f in (..\..\..\..\..\gpMgmt\bin\pythonSrc\ext\PyYAML-*.tar.gz) do tar -xf %%f
-for /D %%d in (PyYAML-*) do copy %%d\lib\yaml\* %GPDB_INSTALL_PATH%\lib\python\yaml
+REM Install PyYAML using pip instead of extracting from tarball
+pip3 install --target=%GPDB_INSTALL_PATH%\lib\python PyYAML==5.4.1
 perl -p -e "s,__VERSION_PLACEHOLDER__,%VERSION%," greenplum-clients.wxs > greenplum-clients-%VERSION%.wxs
 candle.exe -nologo greenplum-clients-%VERSION%.wxs -out greenplum-clients-%VERSION%.wixobj -dSRCDIR=%GPDB_INSTALL_PATH% -dVERSION=%VERSION%
 light.exe -nologo -sval greenplum-clients-%VERSION%.wixobj -out greenplum-clients-x86_64.msi

gpMgmt/bin/.gitignore

@@ -27,3 +27,5 @@
 # ignore anything in ext but keep folder
 ext/*
 !ext/.gitkeep
+# Python sources and temporary install dirs
+pythonSrc/ext/*

gpMgmt/bin/Makefile

@@ -79,26 +79,41 @@ endif
 #
 # Python Libraries
 #
+# Download Python source packages using pip3 download
+# This replaces the previously bundled tar.gz files to comply with Apache Release policy
+#
+MOCK_VERSION=1.0.1
+PYGRESQL_VERSION=5.2
+PSUTIL_VERSION=5.7.0
+PYYAML_VERSION=5.4.1
+
+download-python-deps:
+	@echo "--- Downloading Python dependencies for gpMgmt modules"
+	@mkdir -p $(PYLIB_SRC_EXT)
+	# Download psutil
+	pip3 download --no-deps --no-binary :all: --no-build-isolation psutil==$(PSUTIL_VERSION) --dest $(PYLIB_SRC_EXT)/
+	# Download PyYAML, wheel is for metadata vefiry;
+	# cython is used for building, see https://github.com/apache/cloudberry/issues/1201
+	pip3 install wheel "cython<3.0.0"
+	pip3 download --no-deps --no-binary :all: --no-build-isolation PyYAML==$(PYYAML_VERSION) --dest $(PYLIB_SRC_EXT)/
+	# Download PyGreSQL, the `PATH` is needed to avoid cannot find the pg_config
+	PATH=$(DESTDIR)$(bindir):$$PATH pip3 download --no-deps --no-binary :all: --no-build-isolation PyGreSQL==$(PYGRESQL_VERSION) --dest $(PYLIB_SRC_EXT)/
 
 #
 # PyGreSQL
 #
-PYGRESQL_VERSION=5.2
 PYGRESQL_DIR=PyGreSQL-$(PYGRESQL_VERSION)
-pygresql:
+pygresql: download-python-deps
 	@echo "--- PyGreSQL"
 	cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PYGRESQL_DIR).tar.gz
 	cd $(PYLIB_SRC_EXT)/$(PYGRESQL_DIR)/ && PATH=$(DESTDIR)$(bindir):$$PATH LDFLAGS='$(LDFLAGS) $(PYGRESQL_LDFLAGS)' python3 setup.py build
 	cp -r $(PYLIB_SRC_EXT)/$(PYGRESQL_DIR)/build/lib*-3*/* $(PYLIB_DIR)/
 
-
 #
 # PSUTIL
 #
-PSUTIL_VERSION=5.7.0
 PSUTIL_DIR=psutil-$(PSUTIL_VERSION)
-
-psutil:
+psutil: download-python-deps
 	@echo "--- psutil"
 ifeq "$(findstring $(BLD_ARCH),aix7_ppc_64 )" ""
 	cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PSUTIL_DIR).tar.gz
@@ -109,24 +124,18 @@ endif
 #
 # PYYAML
 #
-PYYAML_VERSION=5.4.1
 PYYAML_DIR=PyYAML-$(PYYAML_VERSION)
 
-pyyaml:
+pyyaml: download-python-deps
 	@echo "--- pyyaml"
 	cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PYYAML_DIR).tar.gz
-	cd $(PYLIB_SRC_EXT)/$(PYYAML_DIR)/ && env -u CC python3 setup.py build
+	cd $(PYLIB_SRC_EXT)/$(PYYAML_DIR)/ && env -u CC CFLAGS="-w" python3 setup.py build
 	cp -r $(PYLIB_SRC_EXT)/$(PYYAML_DIR)/build/lib*-3*/* $(PYLIB_DIR)/
 
 #
 # MOCK SETUP
 #
-MOCK_VERSION=1.0.1
 MOCK_DIR=mock-$(MOCK_VERSION)
-SETUP_TOOLS_VERSION=36.6.0
-PARSE_VERSION=1.8.2
-SETUP_TOOLS_DIR=setuptools-$(SETUP_TOOLS_VERSION)
-PARSE_DIR=parse-$(PARSE_VERSION)
 PYTHONSRC_INSTALL=$(PYLIB_SRC_EXT)/install
 PYTHON_VERSION=$(shell python3 -c "import sys; print ('%s.%s' % (sys.version_info[0:2]))")
 PYTHONSRC_INSTALL_SITE=$(PYLIB_SRC_EXT)/install/lib/python$(PYTHON_VERSION)/site-packages
@@ -140,8 +149,10 @@ $(MOCK_BIN):
 	@if [ "$(UBUNTU_PLATFORM)" = "Ubuntu" ]; then\
        pip3 install mock;\
      else\
-       mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
-	   cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(MOCK_DIR).tar.gz && \
+	   mkdir -p $(PYLIB_SRC_EXT) && \
+	   pip3 download --no-deps --no-binary :all: --no-build-isolation mock==$(MOCK_VERSION) --dest $(PYLIB_SRC_EXT)/ && \
+	   mkdir -p $(PYTHONSRC_INSTALL_SITE) && \
+	   cd $(PYLIB_SRC_EXT)/ && unzip -q $(MOCK_DIR).zip && \
 	   cd $(PYLIB_SRC_EXT)/$(MOCK_DIR)/ && \
 	   PYTHONPATH=$(PYTHONSRC_INSTALL_PYTHON_PATH) python3 setup.py install --prefix $(PYTHONSRC_INSTALL) ; \
 	 fi;
@@ -207,8 +218,8 @@ installcheck: installcheck-bash
 
 clean distclean:
 	rm -rf $(RUFF_BIN) $(SRC)/ruff.txt $(SRC)/.ruff_cache
-	rm -rf $(PYLIB_SRC_EXT)/$(PYGRESQL_DIR)/build
-	rm -rf $(PYLIB_SRC)/$(PYGRESQL_DIR)/build
+	rm -rf $(PYLIB_SRC_EXT)/*.tar.gz $(PYLIB_SRC_EXT)/*.zip $(PYLIB_SRC_EXT)/*.whl
+	rm -rf $(PYLIB_SRC_EXT)/*/
 	rm -rf *.pyc
 	rm -f analyzedbc gpactivatestandbyc gpaddmirrorsc gpcheckcatc \
 		  gpcheckperfc gpcheckresgroupimplc gpchecksubnetcfgc gpconfigc \

gpMgmt/bin/pythonSrc/ext/.gitkeep

@@ -0,0 +1,2 @@
+# This directory is used for downloading Python source packages via pip3 download
+# The downloaded tar.gz files should not be committed to git

gpMgmt/test/README

@@ -29,7 +29,7 @@ make -f Makefile.behave behave flags="--tags smoke --tags mirrors"
 WARNING: The tag boolean logic syntax is different in this version versus
 the version documented online.
 
-From gpdb/gpMgmt/bin/pythonSrc/ext/behave-1.2.4/docs/behave.rst:
+From behave documentation (behave is now installed via pip):
 
 | Tag Expression
 | --------------

python-dependencies.txt

@@ -1,3 +1,3 @@
 psutil==5.7.0
 pygresql==5.2
-pyyaml==5.3.1
+pyyaml==5.4.1