Skip to content

Commit 8bff83d

Browse files
zwarezware
committed
bpo-30450: Pull Windows dependencies from GitHub rather than svn (pythonGH-1783)
The Windows build now depends on Python 3.6 to fetch externals, but it will be downloaded via NuGet (which is downloaded via PowerShell) if it is not available via `py -3.6`. This means the only thing that must be installed on a modern Windows box to do a full build of CPython with all extensions is Visual Studio. (cherry-picked from 51599e2)
1 parent 4cde4bd commit 8bff83d

3 files changed

Lines changed: 141 additions & 66 deletions

File tree

‎PCbuild/get_external.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/usr/bin/env python3
2+
3+
import argparse
4+
import os
5+
import pathlib
6+
import zipfile
7+
from urllib.request import urlretrieve
8+
9+
10+
def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose):
11+
repo = f'cpython-{"bin" if binary else "source"}-deps'
12+
url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip'
13+
reporthook = None
14+
if verbose:
15+
reporthook = print
16+
zip_dir.mkdir(parents=True, exist_ok=True)
17+
filename, headers = urlretrieve(
18+
url,
19+
zip_dir / f'{commit_hash}.zip',
20+
reporthook=reporthook,
21+
)
22+
return filename
23+
24+
25+
def extract_zip(externals_dir, zip_path):
26+
with zipfile.ZipFile(os.fspath(zip_path)) as zf:
27+
zf.extractall(os.fspath(externals_dir))
28+
return externals_dir / zf.namelist()[0].split('/')[0]
29+
30+
31+
def parse_args():
32+
p = argparse.ArgumentParser()
33+
p.add_argument('-v', '--verbose', action='store_true')
34+
p.add_argument('-b', '--binary', action='store_true',
35+
help='Is the dependency in the binary repo?')
36+
p.add_argument('-O', '--organization',
37+
help='Organization owning the deps repos', default='python')
38+
p.add_argument('-e', '--externals-dir', type=pathlib.Path,
39+
help='Directory in which to store dependencies',
40+
default=pathlib.Path(__file__).parent.parent / 'externals')
41+
p.add_argument('tag',
42+
help='tag of the dependency')
43+
return p.parse_args()
44+
45+
46+
def main():
47+
args = parse_args()
48+
zip_path = fetch_zip(
49+
args.tag,
50+
args.externals_dir / 'zips',
51+
org=args.organization,
52+
binary=args.binary,
53+
verbose=args.verbose,
54+
)
55+
final_name = args.externals_dir / args.tag
56+
extract_zip(args.externals_dir, zip_path).replace(final_name)
57+
58+
59+
if __name__ == '__main__':
60+
main()

‎PCbuild/get_externals.bat‎

Lines changed: 66 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -2,51 +2,53 @@
22
setlocal
33
rem Simple script to fetch source for external libraries
44

5-
if not exist "%~dp0..\externals" mkdir "%~dp0..\externals"
6-
pushd "%~dp0..\externals"
5+
if "%PCBUILD%"=="" (set PCBUILD=%~dp0)
6+
if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals)
7+
if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\nuget.exe)
8+
if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl)
79

8-
if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/
10+
set DO_FETCH=true
11+
set DO_CLEAN=false
912

10-
rem Optionally clean up first. Be warned that this can be very destructive!
11-
if not "%1"=="" (
12-
for %%c in (-c --clean --clean-only) do (
13-
if "%1"=="%%c" goto clean
14-
)
15-
goto usage
16-
)
17-
goto fetch
13+
:CheckOpts
14+
if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts
15+
if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts
16+
if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts
17+
if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts
18+
if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts
19+
if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts
20+
if "%~1"=="--clean-only" (set DO_FETCH=false) & goto clean
21+
if "x%~1" NEQ "x" goto usage
1822

23+
if "%DO_CLEAN%"=="false" goto fetch
1924
:clean
2025
echo.Cleaning up external libraries.
21-
for /D %%d in (
22-
bzip2-*
23-
db-*
24-
nasm-*
25-
openssl-*
26-
tcl-*
27-
tcltk*
28-
tk-*
29-
tix-*
30-
sqlite-*
31-
xz-*
32-
) do (
33-
echo.Removing %%d
34-
rmdir /s /q %%d
35-
)
36-
if "%1"=="--clean-only" (
37-
goto end
26+
if exist "%EXTERNALS_DIR%" (
27+
rem Sometimes this fails the first time; try it twice
28+
rmdir /s /q "%EXTERNALS_DIR%" || rmdir /s /q "%EXTERNALS_DIR%"
3829
)
3930

31+
if "%DO_FETCH%"=="false" goto end
4032
:fetch
41-
rem Fetch current versions
42-
43-
svn --version > nul 2>&1
44-
if ERRORLEVEL 9009 (
45-
echo.svn.exe must be on your PATH.
46-
echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the
47-
echo.command line tools option.
48-
popd
49-
exit /b 1
33+
34+
if "%ORG%"=="" (set ORG=python)
35+
36+
if "%PYTHON_FOR_BUILD%"=="" (
37+
echo Checking for installed python...
38+
py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6)
39+
)
40+
if "%PYTHON_FOR_BUILD%"=="" (
41+
if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%"
42+
if NOT exist "%NUGET%" (
43+
echo Downloading nuget...
44+
rem NB: Must use single quotes around NUGET here, NOT double!
45+
rem Otherwise, a space in the path would break things
46+
powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%'
47+
)
48+
echo Installing Python via nuget...
49+
"%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%"
50+
rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
51+
set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe"
5052
)
5153

5254
echo.Fetching external libraries...
@@ -57,51 +59,55 @@ rem files in both this dir and PC\VS9.0
5759
set libraries=
5860
set libraries=%libraries% bzip2-1.0.6
5961
if NOT "%IncludeBsddb%"=="false" set libraries=%libraries% db-4.7.25.0
60-
if NOT "%IncludeSSL%"=="false" set libraries=%libraries% nasm-2.11.06
6162
if NOT "%IncludeSSL%"=="false" set libraries=%libraries% openssl-1.0.2k
6263
set libraries=%libraries% sqlite-3.8.11.0
6364
if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tcl-8.5.15.0
6465
if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tk-8.5.15.0
6566
if NOT "%IncludeTkinter%"=="false" set libraries=%libraries% tix-8.4.3.5
6667

6768
for %%e in (%libraries%) do (
68-
if exist %%e (
69+
if exist "%EXTERNALS_DIR%\%%e" (
6970
echo.%%e already exists, skipping.
7071
) else (
7172
echo.Fetching %%e...
72-
svn export -q %SVNROOT%%%e
73+
%PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -O %ORG% %%e
74+
)
75+
)
76+
77+
echo.Fetching external binaries...
78+
79+
set binaries=
80+
set binaries=%binaries%
81+
if NOT "%IncludeSSL%"=="false" set binaries=%binaries% nasm-2.11.06
82+
83+
for %%b in (%binaries%) do (
84+
if exist "%EXTERNALS_DIR%\%%b" (
85+
echo.%%b already exists, skipping.
86+
) else (
87+
echo.Fetching %%b...
88+
%PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -b -O %ORG% %%b
7389
)
7490
)
7591

92+
echo Finished.
7693
goto end
7794

7895
:usage
79-
echo.invalid argument: %1
80-
echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ]
96+
echo.Valid options: -c, --clean, --clean-only, --organization, --python,
97+
echo.--no-tkinter, --no-openssl
8198
echo.
82-
echo.Pull all sources necessary for compiling optional extension modules
83-
echo.that rely on external libraries. Requires svn.exe to be on your PATH
84-
echo.and pulls sources from %SVNROOT%.
99+
echo.Pull all sources and binaries necessary for compiling optional extension
100+
echo.modules that rely on external libraries.
85101
echo.
86-
echo.Use the -c or --clean option to clean up all external library sources
87-
echo.before pulling in the current versions.
102+
echo.The --organization option determines which github organization to download
103+
echo.from, the --python option determines which Python 3.6+ interpreter to use
104+
echo.with PCbuild\get_external.py.
105+
echo.
106+
echo.Use the -c or --clean option to remove the entire externals directory.
88107
echo.
89108
echo.Use the --clean-only option to do the same cleaning, without pulling in
90109
echo.anything new.
91110
echo.
92-
echo.Only the first argument is checked, all others are ignored.
93-
echo.
94-
echo.**WARNING**: the cleaning options unconditionally remove any directory
95-
echo.that is a child of
96-
echo. %CD%
97-
echo.and matches wildcard patterns beginning with bzip2-, db-, nasm-, openssl-,
98-
echo.tcl-, tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential
99-
echo.to be very destructive if you are not aware of what it is doing. Use with
100-
echo.caution!
101-
popd
102111
exit /b -1
103112

104-
105113
:end
106-
echo Finished.
107-
popd

‎PCbuild/readme.txt‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@ Quick Start Guide
44
1. Install Microsoft Visual Studio 2008, any edition.
55
2. Install Microsoft Visual Studio 2010, any edition, or Windows SDK 7.1
66
and any version of Microsoft Visual Studio newer than 2010.
7-
3. Install Subversion, and make sure 'svn.exe' is on your PATH.
8-
4. Run "build.bat -e" to build Python in 32-bit Release configuration.
9-
5. (Optional, but recommended) Run the test suite with "rt.bat -q".
7+
2a. Optionally install Python 3.6 or later. If not installed,
8+
get_externals.bat (build.bat -e) will download and use Python via
9+
NuGet.
10+
3. Run "build.bat -e" to build Python in 32-bit Release configuration.
11+
4. (Optional, but recommended) Run the test suite with "rt.bat -q".
1012

1113

1214
Building Python using MSVC 9.0 via MSBuild
@@ -250,9 +252,16 @@ order to download the relevant source files for each project before they
250252
can be built. However, a simple script is provided to make this as
251253
painless as possible, called "get_externals.bat" and located in this
252254
directory. This script extracts all the external sub-projects from
253-
http://svn.python.org/projects/external
254-
via Subversion (so you'll need svn.exe on your PATH) and places them
255-
in ..\externals (relative to this directory).
255+
https://github.com/python/cpython-source-deps
256+
and
257+
https://github.com/python/cpython-bin-deps
258+
via a Python script called "get_external.py", located in this directory.
259+
If Python 3.6 or later is not available via the "py.exe" launcher, the
260+
path or command to use for Python can be provided in the PYTHON_FOR_BUILD
261+
environment variable, or get_externals.bat will download the latest
262+
version of NuGet and use it to download the latest "pythonx86" package
263+
for use with get_external.py. Everything downloaded by these scripts is
264+
stored in ..\externals (relative to this directory).
256265

257266
It is also possible to download sources from each project's homepage,
258267
though you may have to change folder names or pass the names to MSBuild

0 commit comments

Comments
 (0)