Batch generation

.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1

@@ -0,0 +1,65 @@
+param (
+    [string]$MatrixKey,
+    [string]$RepoRoot
+)
+
+Write-Host "Matrix Key: $MatrixKey"
+
+$generateTargetsOutputFile = Join-Path $RepoRoot "artifacts" "generateTargets.json"
+$generateTargets = Get-Content -Path $generateTargetsOutPutFile -Raw | ConvertFrom-Json
+
+$moduleGroup = $generateTargets.$MatrixKey
+$sortedModuleNames = $moduleGroup.PSObject.Properties.Name | Sort-Object
+
+foreach ($moduleName in $sortedModuleNames) {
+    $subModules = $moduleGroup.$moduleName
+    Write-Host "Module Name: $moduleName"
+    foreach ($subModule in $subModules) {
+        Write-Host "  SubModule: $subModule"
+    }
+}
+
+# $AutorestOutputDir = Join-Path $RepoRoot "artifacts" "autorest"
+# New-Item -ItemType Directory -Force -Path $AutorestOutputDir
+
+# $sourceDirectory = Join-Path $RepoRoot "src"
+# # $buildModulesPath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildModules.ps1'
+# $results = @()
+
+# foreach ($module in $moduleGroup) {
+#     $startTime = Get-Date
+#     $result = @{
+#         Module = $module
+#         Status = "Success"
+#         DurationSeconds = 0
+#         Error = ""
+#     }
+#     try {
+#         Write-Host "Building module: $module"
+#         #TODO(Bernard) Remove log after test
+#         if ($module -eq 'Communication') {
+#             throw "Module '$module' is not allowed for this operation."
+#         }
+#         # & $buildModulesPath -TargetModule $module -ForceRegenerate -InvokedByPipeline
+#         $moduleRootSource = Join-Path $sourceDirectory $ModuleRootName
+#         $moduleRootGenerated = Join-Path $generatedDirectory $ModuleRootName
+
+#     } catch {
+#         Write-Warning "Failed to build module: $module"
+#         $result.Status = "Failed"
+#         $result.Error = $_.Exception.Message
+#     } finally {
+#         $endTime = Get-Date
+#         $result.DurationSeconds = ($endTime - $startTime).TotalSeconds
+#         $results += $result
+#     }
+# }
+
+# git add .
+# $patchPath = Join-Path $ArtifactRoot "changed.patch"
+# git diff --cached > $patchPath
+
+# $reportPath = Join-Path $ArtifactRoot "BuildReport-$MatrixKey.json"
+# $results | ConvertTo-Json -Depth 3 | Out-File -FilePath $reportPath -Encoding utf8
+
+# Write-Host "Build report written to $reportPath"

.azure-pipelines/PipelineSteps/BatchGeneration/create-branch.ps1

@@ -0,0 +1,22 @@
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+    [string]$Owner,
+    [string]$Repo,
+    [string]$BaseBranch,
+    [string]$NewBranch,
+    [string]$Token
+)
+
+$headers = @{ Authorization = "Bearer $Token"; "User-Agent" = "ADO-Pipeline" }
+$branchInfo = Invoke-RestMethod -Uri "https://api.github.com/repos/$Owner/$Repo/git/ref/heads/$BaseBranch" -Headers $headers
+$sha = $branchInfo.object.sha
+
+$body = @{
+    ref = "refs/heads/$NewBranch"
+    sha = $sha
+} | ConvertTo-Json
+
+Invoke-RestMethod -Uri "https://api.github.com/repos/$Owner/$Repo/git/refs" `
+                -Method Post -Headers $headers -Body $body -ContentType "application/json"
+
+Write-Host "Created branch '$NewBranch' from '$BBaseBranch'"

.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1

@@ -0,0 +1,89 @@
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+    [int]$MaxParalleAnalyzeJobs = 3,
+    [int]$MaxParalleTestWindowsJobs = 3,
+    [int]$MaxParalleTestLinuxJobs = 3,
+    [int]$MaxParalleTestMacJobs = 3,
+    [string[]]$ChangedFiles
+)
+
+$autorestFolders = @{}
+for ($i = 0; $i -lt $ChangedFiles.Count; $i++) {
+    if ($ChangedFiles[$i] -match '^(src|generated)/([^/]+)/([^/]+\.autorest)/') {
+        $parent = $Matches[2]
+        $child = $Matches[3]
+        $key = "$parent/$child"
+
+        $autorestFolders[$key] = $true
+    }
+}
+
+$changedSubModules = $autorestFolders.Keys
+# TODO(Bernard) Remove test data after test
+# $changedSubModules = @("A", "B", "C", "D", "E", "F", "G")
+Write-Host "Chagned sub modules: "
+foreach ($subModule in $changedSubModules) {
+    Write-Host $subModule
+}
+
+function Split-List {
+    param (
+        [array]$subModules,
+        [int]$maxParallelJobs
+    )
+
+    $count = $subModules.Count
+    $n = [Math]::Min($count, $maxParallelJobs)
+
+    if ($n -eq 0) {
+        return @()
+    }
+
+    $result = @()
+
+    for ($i = 0; $i -lt $n; $i++) {
+        $result += ,@()
+    }
+
+    for ($i = 0; $i -lt $count; $i++) {
+        $groupIndex = $i % $n
+        $result[$groupIndex] += $subModules[$i]
+    }
+
+    return ,$result
+}
+
+function Write-Matrix {
+    param (
+        [string]$variableName,
+        [array]$groupedSubModules
+    )
+
+    $index = 0
+    foreach ($subModules in $groupedSubModules) {
+        $moduleNamesStr = $subModules -join ','
+        $key = ($index + 1).ToString() + "-" + $subModules.Count
+        $MatrixStr = "$MatrixStr,'$key':{'Target':'$moduleNamesStr','MatrixKey':'$key'}"
+        $index++
+    }
+
+    if ($MatrixStr -and $MatrixStr.Length -gt 1) {
+        $MatrixStr = $MatrixStr.Substring(1)
+    }
+    Write-Host "##vso[task.setVariable variable=$variableName;isOutput=true]{$MatrixStr}"
+    Write-Host "variable=$variableName; value=$MatrixStr"
+    }
+
+$groupedAnalyzeModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleAnalyzeJobs
+Write-Matrix -variableName 'AnalyzeTargets' -groupedSubModules $groupedAnalyzeModules
+
+$groupedTestWindowsModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestWindowsJobs
+Write-Matrix -variableName 'TestWindowsTargets' -groupedSubModules $groupedTestWindowsModules
+
+$groupedTestLinuxModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestLinuxJobs
+Write-Matrix -variableName 'TestLinuxTargets' -groupedSubModules $groupedTestLinuxModules
+
+# $groupedTestMacModules = Split-List -subModules $changedSubModules -maxParallelJobs $MaxParalleTestMacJobs
+# Write-Matrix -variableName 'TestMacTargets' -groupedSubModules $groupedTestMacModules
+
+Write-Host "##vso[task.setVariable variable=TestMacTargets;isOutput=true]{}"

.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1

@@ -0,0 +1,116 @@
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+    [string]$RepoRoot,
+    [int]$MaxParallelJobs = 3
+)
+
+$srcPath = Join-Path $RepoRoot 'src'
+function Get-SubModuleWithAutorestV4 {
+    param (
+        [string]$srcPath
+    )
+
+    $result = @{}
+
+    Get-ChildItem -Path $srcPath -Directory | ForEach-Object {
+        $module = $_
+
+        Get-ChildItem -Path $module.FullName -Directory | Where-Object { 
+            $_.Name -like '*.autorest'
+        } | ForEach-Object {
+            $subModule = $_
+
+            $readmePath = Join-Path $subModule.FullName 'README.md'
+
+            if (Test-Path $readmePath) {
+                $readmeContent = Get-Content -Path $readmePath -Raw
+
+                if ($readmeContent -notmatch 'use-extension:\s+"@autorest/powershell":\s+"3.x"') {
+                    if ($result.ContainsKey($module.Name)) {
+                        $result[$module.Name] += $subModule.Name
+                    } else {
+                        $result[$module.Name] = @($subModule.Name) 
+                    }
+                }
+            }
+        }
+    }
+
+    return $result
+}
+# TODO(Bernard): Use real function after test
+# $modules = Get-SubModuleWithAutorestV4 -srcPath $srcPath
+$modules = @{
+    "DeviceRegistry" = @("DeviceRegistry.Autorest")
+    "ArcGateway" = @("ArcGateway.Autorest")
+    "Chaos" = @("Chaos.Autorest")
+    "Cdn" = @("Cdn.Autorest")
+    "Communication" = @("EmailService.Autorest", "EmailServicedata.Autorest")
+    "Astro" = @("Astro.Autorest")
+    "ImageBuilder" = @("ImageBuilder.Autorest")
+}
+$modules = $modules.GetEnumerator() | ForEach-Object {
+    [PSCustomObject]@{
+        ModuleName = $_.Key
+        SubModules = ($_.Value | Sort-Object)
+    }
+} | Sort-Object -Property ModuleName
+
+Write-Host "Total matched modules: $($modules.Count)"
+
+function Group-List {
+    param (
+        [array]$modules,
+        [int]$maxParallelJobs
+    )
+
+    $count = $modules.Count
+    $n = [Math]::Min($count, $maxParallelJobs)
+    if ($n -eq 0) {
+        return @()
+    }
+
+    $result = @()
+    $sizePerGroup = [Math]::Ceiling($count / $n)
+
+    for ($i = 0; $i -lt $count; $i += $sizePerGroup) {
+        $group = $modules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
+        $result += ,$group
+    }
+
+    return $result
+}
+
+$groupedModules = Group-List -modules $modules -maxParallelJobs $MaxParallelJobs
+Write-Host "Total module groups: $($groupedModules.Count)"
+
+$index = 0
+$generateTargets = @{}
+foreach ($moduleGroup in $groupedModules) {
+    Write-Host "##[group]Prepareing module group $($index + 1)"
+    $mergedModules = @{}
+    foreach ($moduleObj in $moduleGroup) {
+        Write-Host "Module $($moduleObj.ModuleName): $($moduleObj.SubModules -join ',')"
+        $mergedModules[$moduleObj.ModuleName] = @($moduleObj.SubModules)
+        $subIndex++
+    }
+
+    $key = ($index + 1).ToString() + "-" + $moduleGroup.Count
+    $generateTargets[$key] = $mergedModules
+    $MatrixStr = "$MatrixStr,'$key':{'MatrixKey':'$key'}"
+    Write-Host "##[endgroup]"
+    Write-Host
+    $index++
+}
+
+$generateTargetsOutputDir = Join-Path $RepoRoot "artifacts"
+if (-not (Test-Path -Path $generateTargetsOutputDir)) {
+    New-Item -ItemType Directory -Path $generateTargetsOutputDir
+}
+$generateTargetsOutputFile = Join-Path $generateTargetsOutputDir "generateTargets.json"
+$generateTargets | ConvertTo-Json -Depth 10 | Out-File -FilePath $generateTargetsOutputFile -Encoding utf8
+
+if ($MatrixStr -and $MatrixStr.Length -gt 1) {
+    $MatrixStr = $MatrixStr.Substring(1)
+}
+Write-Host "##vso[task.setVariable variable=generateTargets;isOutput=true]{$MatrixStr}"

.azure-pipelines/PipelineSteps/BatchGeneration/test-module.ps1

@@ -0,0 +1,45 @@
+param (
+    [string]$MatrixKey,
+    [string]$Target,
+    [string]$TestEnvName,
+    [string]$RepoRoot,
+    [string]$ArtifactRoot
+)
+
+Write-Host "Matrix Key: $MatrixKey"
+Write-Host "Test $($TestEnvName): $Target"
+
+$modules = $Target -split ','
+$results = @()  
+
+foreach ($module in $modules) {
+    $startTime = Get-Date
+    $result = @{
+        Module = $module
+        Status = "Success"
+        DurationSeconds = 0
+        Error = ""
+    }
+
+    try {
+        Write-Host "Testing module: $module"
+        $subModulePath = Join-Path $RepoRoot 'src' $module
+        # TODO(Bernard) Remove log after test
+        Write-Host "Sub module path: $subModulePath"
+        Set-Location -Path $subModulePath
+        & ".\test-module.ps1"  
+    }
+    catch {
+        Write-Warning "Failed to test module: $module"
+        $result.Status = "Failed"
+        $result.Error = $_.Exception.Message
+    }
+    finally {
+        $endTime = Get-Date
+        $result.DurationSeconds = ($endTime - $startTime).TotalSeconds
+        $results += $result
+    }
+}
+
+$reportPath = Join-Path $ArtifactRoot "Test$($TestEnvName)Report-$MatrixKey.json"
+$results | ConvertTo-Json -Depth 3 | Out-File -FilePath $reportPath -Encoding utf8