Skip to content

[windows] Consolidate dependency infos in build.ps1 (NFC) #77945

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

[windows] Consolidate dependency infos in build.ps1 (NFC) #77945

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
61fd0b1
Consolidate dependency infos
weliveindetail Dec 4, 2024
1a64db3
Consolidate Python module installation
weliveindetail Dec 4, 2024
16db363
fixup! Consolidate Python module installation
weliveindetail Dec 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
187 changes: 106 additions & 81 deletions utils/build.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,6 @@ param(
[string] $PinnedBuild = "",
[string] $PinnedSHA256 = "",
[string] $PinnedVersion = "",
[string] $PythonVersion = "3.9.10",
[string] $AndroidNDKVersion = "r26b",
[string] $WinSDKVersion = "",
[switch] $Android = $false,
Expand Down Expand Up @@ -163,22 +162,6 @@ $env:SDKROOT = ""
$BuildArchName = $env:PROCESSOR_ARCHITEW6432
if ($null -eq $BuildArchName) { $BuildArchName = $env:PROCESSOR_ARCHITECTURE }

if ($PinnedBuild -eq "") {
switch ($BuildArchName) {
"AMD64" {
$PinnedBuild = "https://download.swift.org/swift-5.10.1-release/windows10/swift-5.10.1-RELEASE/swift-5.10.1-RELEASE-windows10.exe"
$PinnedSHA256 = "3027762138ACFA1BBE3050FF6613BBE754332E84C9EFA5C23984646009297286"
$PinnedVersion = "5.10.1"
}
"ARM64" {
$PinnedBuild = "https://download.swift.org/development/windows10-arm64/swift-DEVELOPMENT-SNAPSHOT-2024-07-02-a/swift-DEVELOPMENT-SNAPSHOT-2024-07-02-a-windows10-arm64.exe"
$PinnedSHA256 = "037BDBF9D1A1A99D7156584948870A8A958FD27CC4FF5711691CC0A76F2E88F5"
$PinnedVersion = "0.0.0"
}
default { throw "Unsupported processor architecture" }
}
}

# Store the revision zero variant of the Windows SDK version (no-op if unspecified)
$WindowsSDKMajorMinorBuildMatch = [Regex]::Match($WinSDKVersion, "^\d+\.\d+\.\d+")
$WinSDKVersionRevisionZero = if ($WindowsSDKMajorMinorBuildMatch.Success) { $WindowsSDKMajorMinorBuildMatch.Value + ".0" } else { "" }
Expand All @@ -188,8 +171,70 @@ $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.e
$VSInstallRoot = & $vswhere -nologo -latest -products "*" -all -prerelease -property installationPath
$msbuild = "$VSInstallRoot\MSBuild\Current\Bin\$BuildArchName\MSBuild.exe"

# Hoist to global scope as this is used in two sites.
# Update package hashes when changing versions
$WiXVersion = "4.0.5"
$PythonVersion = "3.9.10"
$WinFlexBisonVersion = "2.5.25"

$Components = @{
WiX = @{
URL = "https://www.nuget.org/api/v2/package/wix/$WiXVersion"
Hash = "DF9BDB347183716F82EFE2CECB8C54BB3554AA907A69F47A41741D6FA4D0A754"
}
PythonAMD64 = @{
URL = "https://www.nuget.org/api/v2/package/python/$PythonVersion"
Hash = "ac43b491e9488ac926ed31c5594f0c9409a21ecbaf99dc7a93f8c7b24cf85867"
}
PythonARM64 = @{
URL = "https://www.nuget.org/api/v2/package/pythonarm64/$PythonVersion"
Hash = "429ada77e7f30e4bd8ff22953a1f35f98b2728e84c9b1d006712561785641f69"
}
PinnedSwiftAMD64 = @{
URL = "https://download.swift.org/swift-5.10.1-release/windows10/swift-5.10.1-RELEASE/swift-5.10.1-RELEASE-windows10.exe"
Hash = "3027762138ACFA1BBE3050FF6613BBE754332E84C9EFA5C23984646009297286"
Version = "5.10.1"
}
PinnedSwiftARM64 = @{
URL = "https://download.swift.org/development/windows10-arm64/swift-DEVELOPMENT-SNAPSHOT-2024-07-02-a/swift-DEVELOPMENT-SNAPSHOT-2024-07-02-a-windows10-arm64.exe"
Hash = "037BDBF9D1A1A99D7156584948870A8A958FD27CC4FF5711691CC0A76F2E88F5"
Version = "0.0.0"
}
PinnedSwiftCustom = @{
URL = $PinnedBuild
Hash = $PinnedSHA256
Version = $PinnedVersion
}
NDK = @{
URL = "https://dl.google.com/android/repository/android-ndk-r26b-windows.zip"
Hash = "A478D43D4A45D0D345CDA6BE50D79642B92FB175868D9DC0DFC86181D80F691E"
}
WinFlexBison = @{
URL = "https://github.com/lexxmark/winflexbison/releases/download/v$WinFlexBisonVersion/win_flex_bison-$WinFlexBisonVersion.zip"
Hash = "8D324B62BE33604B2C45AD1DD34AB93D722534448F55A16CA7292DE32B6AC135"
}
packaging = @{
URL = "https://files.pythonhosted.org/packages/08/aa/cc0199a5f0ad350994d660967a8efb233fe0416e4639146c089643407ce6/packaging-24.1-py3-none-any.whl"
Hash = "5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"
}
distutils = @{
# 'setuptools' provides 'distutils' module for Python 3.12+, required for SWIG support
# https://github.com/swiftlang/llvm-project/issues/9289
URL = "https://files.pythonhosted.org/packages/ff/ae/f19306b5a221f6a436d8f2238d5b80925004093fa3edea59835b514d9057/setuptools-75.1.0-py3-none-any.whl"
Hash = "35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2"
}
}

if ($PinnedBuild -eq "") {
$PinnedSwift = "PinnedSwift$BuildArchName"
if ($Components[$PinnedSwift] -eq $null) {
throw "Unsupported processor architecture"
}
} else {
$PinnedSwift = "PinnedSwiftCustom"
if ($Components[$PinnedSwift].Version -eq "") {
throw "PinnedVersion must be set"
}
}

# Avoid $env:ProgramFiles in case this script is running as x86
$UnixToolsBinDir = "$env:SystemDrive\Program Files\Git\usr\bin"
Expand Down Expand Up @@ -375,7 +420,7 @@ function Get-HostSwiftSDK() {
}

$NugetRoot = "$BinaryCache\nuget"
$PinnedToolchain = [IO.Path]::GetFileNameWithoutExtension($PinnedBuild)
$PinnedToolchain = [IO.Path]::GetFileNameWithoutExtension($Components[$PinnedSwift].URL)

$LibraryRoot = "$ImageRoot\Library"

Expand Down Expand Up @@ -630,22 +675,26 @@ function Fetch-Dependencies {

$WebClient = New-Object Net.WebClient

function DownloadAndVerify($URL, $Destination, $Hash) {
function DownloadAndVerify($Component, $Destination) {
if (Test-Path $Destination) {
return
}
$Comp = $Components[$Component]
if ($Comp -eq $null) {
throw "Unknown component requested"
}

Write-Output "$Destination not found. Downloading ..."
if ($ToBatch) {
Write-Output "md `"$(Split-Path -Path $Destination -Parent)`""
Write-Output "curl.exe -sL $URL -o $Destination"
Write-Output "(certutil -HashFile $Destination SHA256) == $Hash || (exit /b)"
Write-Output "curl.exe -sL $($Comp.URL) -o $Destination"
Write-Output "(certutil -HashFile $Destination SHA256) == $($Comp.Hash) || (exit /b)"
} else {
New-Item -ItemType Directory (Split-Path -Path $Destination -Parent) -ErrorAction Ignore | Out-Null
$WebClient.DownloadFile($URL, $Destination)
$WebClient.DownloadFile($Comp.URL, $Destination)
$SHA256 = Get-FileHash -Path $Destination -Algorithm SHA256
if ($SHA256.Hash -ne $Hash) {
throw "SHA256 mismatch ($($SHA256.Hash) vs $Hash)"
if ($SHA256.Hash -ne $Comp.Hash) {
throw "SHA256 mismatch ($($SHA256.Hash) vs $($Comp.Hash))"
}
}
}
Expand Down Expand Up @@ -710,95 +759,74 @@ function Fetch-Dependencies {
Get-ChildItem "$BinaryCache\toolchains\WixAttachedContainer" -Filter "*.msi" | ForEach-Object {
$LogFile = [System.IO.Path]::ChangeExtension($_.Name, "log")
$TARGETDIR = if ($_.Name -eq "rtl.msi") { "$BinaryCache\toolchains\$ToolchainName\LocalApp\Programs\Swift\Runtimes\$(Get-PinnedToolchainVersion)\usr\bin" } else { "$BinaryCache\toolchains\$ToolchainName" }
Invoke-Program -OutNull msiexec.exe /lvx! $BinaryCache\toolchains\$LogFile /qn /a $BinaryCache\toolchains\WixAttachedContainer\$($_.Name) ALLUSERS=0 TARGETDIR=$TARGETDIR
Invoke-Program -OutNull msiexec.exe /lvx! $BinaryCache\toolchains\$LogFile /qn /a $BinaryCache\toolchains\WixAttachedContainer\$($_.Name) ALLUSERS=0 TARGETDIR=$TARGETDIR
}
}

if ($SkipBuild -and $SkipPackaging) { return }

$WiXURL = "https://www.nuget.org/api/v2/package/wix/$WiXVersion"
$WiXHash = "DF9BDB347183716F82EFE2CECB8C54BB3554AA907A69F47A41741D6FA4D0A754"
DownloadAndVerify $WixURL "$BinaryCache\WiX-$WiXVersion.zip" $WiXHash
DownloadAndVerify "WiX" "$BinaryCache\WiX-$WiXVersion.zip"
Extract-ZipFile WiX-$WiXVersion.zip $BinaryCache WiX-$WiXVersion

if ($SkipBuild) { return }

DownloadAndVerify $PinnedBuild "$BinaryCache\$PinnedToolchain.exe" $PinnedSHA256
DownloadAndVerify $PinnedSwift "$BinaryCache\$PinnedToolchain.exe"

# TODO(compnerd) stamp/validate that we need to re-extract
New-Item -ItemType Directory -ErrorAction Ignore $BinaryCache\toolchains | Out-Null
Extract-Toolchain "$PinnedToolchain.exe" $BinaryCache $PinnedToolchain

function Download-Python($ArchName) {
$PythonAMD64URL = "https://www.nuget.org/api/v2/package/python/$PythonVersion"
$PythonAMD64Hash = "ac43b491e9488ac926ed31c5594f0c9409a21ecbaf99dc7a93f8c7b24cf85867"

$PythonARM64URL = "https://www.nuget.org/api/v2/package/pythonarm64/$PythonVersion"
$PythonARM64Hash = "429ada77e7f30e4bd8ff22953a1f35f98b2728e84c9b1d006712561785641f69"

DownloadAndVerify (Get-Variable -Name "Python${ArchName}URL").Value $BinaryCache\Python$ArchName-$PythonVersion.zip (Get-Variable -Name "Python${ArchName}Hash").Value

DownloadAndVerify "Python$ArchName" "$BinaryCache\Python$ArchName-$PythonVersion.zip"
if (-not $ToBatch) {
Extract-ZipFile Python$ArchName-$PythonVersion.zip $BinaryCache Python$ArchName-$PythonVersion
}
}

function Ensure-PythonModules($Python) {
# First ensure pip is installed, else bootstrap it
try {
Invoke-Program -OutNull $Python -m pip *> $null
} catch {
Write-Output "Installing pip ..."
Invoke-Program -OutNull $Python '-I' -m ensurepip -U --default-pip
}
# 'packaging' is required for building LLVM 18+
try {
Invoke-Program -OutNull $Python -c 'import packaging' *> $null
} catch {
$WheelURL = "https://files.pythonhosted.org/packages/08/aa/cc0199a5f0ad350994d660967a8efb233fe0416e4639146c089643407ce6/packaging-24.1-py3-none-any.whl"
$WheelHash = "5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"
DownloadAndVerify $WheelURL "$BinaryCache\python\packaging-24.1-py3-none-any.whl" $WheelHash
Write-Output "Installing 'packaging-24.1-py3-none-any.whl' ..."
Invoke-Program -OutNull $Python '-I' -m pip install "$BinaryCache\python\packaging-24.1-py3-none-any.whl" --disable-pip-version-check
Download-Python $HostArchName
if ($IsCrossCompiling) {
Download-Python $BuildArchName
}

# Ensure pip is installed, else bootstrap it
try {
Invoke-Program -OutNull $(Get-PythonExecutable) -m pip *> $null
} catch {
Write-Output "Installing pip ..."
Invoke-Program -OutNull $(Get-PythonExecutable) '-I' -m ensurepip -U --default-pip
}

function Ensure-PythonModule($Name) {
$Info = $Components[$Name]
if ($Info -eq $null) {
throw "Unknown component requested"
}
# 'setuptools' provides 'distutils' module for Python 3.12+, required for SWIG support
# https://github.com/swiftlang/llvm-project/issues/9289
$CanonicalName = [IO.Path]::GetFileNameWithoutExtension($Info.URL)

try {
Invoke-Program -OutNull $Python -c 'import distutils' *> $null
Invoke-Program -OutNull $(Get-PythonExecutable) -c 'import $name' *> $null
} catch {
$WheelURL = "https://files.pythonhosted.org/packages/ff/ae/f19306b5a221f6a436d8f2238d5b80925004093fa3edea59835b514d9057/setuptools-75.1.0-py3-none-any.whl"
$WheelHash = "35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2"
DownloadAndVerify $WheelURL "$BinaryCache\python\setuptools-75.1.0-py3-none-any.whl" $WheelHash
Write-Output "Installing 'setuptools-75.1.0-py3-none-any.whl' ..."
Invoke-Program -OutNull $Python '-I' -m pip install "$BinaryCache\python\setuptools-75.1.0-py3-none-any.whl" --disable-pip-version-check
DownloadAndVerify $Name "$BinaryCache\python\$CanonicalName.whl"
Write-Output "Installing '$CanonicalName.whl' ..."
Invoke-Program -OutNull $(Get-PythonExecutable) '-I' -m pip install "$BinaryCache\python\$CanonicalName.whl" --disable-pip-version-check
}
}

Download-Python $HostArchName
if ($IsCrossCompiling) {
Download-Python $BuildArchName
}
# Ensure Python modules that are required as host build tools
Ensure-PythonModules "$(Get-PythonExecutable)"
Ensure-PythonModule "packaging"
Ensure-PythonModule "distutils"

if ($Android) {
# Only a specific NDK version is supported right now.
if ($AndroidNDKVersion -ne "r26b") {
throw "Unsupported Android NDK version"
}
$NDKURL = "https://dl.google.com/android/repository/android-ndk-r26b-windows.zip"
$NDKHash = "A478D43D4A45D0D345CDA6BE50D79642B92FB175868D9DC0DFC86181D80F691E"
DownloadAndVerify $NDKURL "$BinaryCache\android-ndk-$AndroidNDKVersion-windows.zip" $NDKHash

DownloadAndVerify "NDK" "$BinaryCache\android-ndk-$AndroidNDKVersion-windows.zip"
Extract-ZipFile -ZipFileName "android-ndk-$AndroidNDKVersion-windows.zip" -BinaryCache $BinaryCache -ExtractPath "android-ndk-$AndroidNDKVersion" -CreateExtractPath $false
}

if ($IncludeDS2) {
$WinFlexBisonVersion = "2.5.25"
$WinFlexBisonURL = "https://github.com/lexxmark/winflexbison/releases/download/v$WinFlexBisonVersion/win_flex_bison-$WinFlexBisonVersion.zip"
$WinFlexBisonHash = "8D324B62BE33604B2C45AD1DD34AB93D722534448F55A16CA7292DE32B6AC135"
DownloadAndVerify $WinFlexBisonURL "$BinaryCache\win_flex_bison-$WinFlexBisonVersion.zip" $WinFlexBisonHash

DownloadAndVerify "WinFlexBison" "$BinaryCache\win_flex_bison-$WinFlexBisonVersion.zip"
Extract-ZipFile -ZipFileName "win_flex_bison-$WinFlexBisonVersion.zip" -BinaryCache $BinaryCache -ExtractPath "win_flex_bison"
}

Expand Down Expand Up @@ -851,10 +879,7 @@ function Get-PinnedToolchainRuntime() {
}

function Get-PinnedToolchainVersion() {
if (Test-Path variable:PinnedVersion) {
return $PinnedVersion
}
throw "PinnedVersion must be set"
$Components[$PinnedSwift].Version
}

function TryAdd-KeyValue([hashtable]$Hashtable, [string]$Key, [string]$Value) {
Expand Down