Batch generation

Author: Pan-Qi

.azure-pipelines/PipelineSteps/BatchGeneration/prepare.ps1

@@ -0,0 +1,81 @@
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+    [string]$RepoRoot,
+    [int]$MaxParallelJobs = 3
+)
+
+$moduleRoot = Join-Path $RepoRoot 'src'
+$subModules = @()
+
+Get-ChildItem -Path $moduleRoot -Directory | ForEach-Object {
+    $module = $_
+    Get-ChildItem -Path $module.FullName -Directory | Where-Object {
+        $_.Name -like '*.autorest'
+    } | ForEach-Object {
+        $sub_module = $_
+        $subModules += ,@($module.Name, $sub_module.Name)
+    }
+}
+
+$subModules = @(
+    # V3
+    @("Cdn","Cdn.Autorest"),
+    @("ImageBuilder", "ImageBuilder.Autorest"),
+
+    # V4
+    @("Chaos", "Chaos.Autorest"),
+    @("DeviceRegistry", "DeviceRegistry.Autorest"),
+    @("Astro", "Astro.Autorest"),
+    
+    # V4 Multi sub-modules
+    @("Communication","EmailService.Autorest"),
+    @("Communication", "EmailServicedata.Autorest")
+)
+
+Write-Host "Total matched sub modules: $($subModules.Count)"
+
+function Split-List {
+    param (
+        [array]$subModules,
+        [int]$maxParallelJobs
+    )
+
+    $count = $subModules.Count
+    $n = [Math]::Min($count, $maxParallelJobs)
+    if ($n -eq 0) {
+        return @()
+    }
+
+    $result = @()
+    $sizePerGroup = [Math]::Ceiling($count / $n)
+
+    for ($i = 0; $i -lt $count; $i += $sizePerGroup) {
+        $group = $subModules[$i..([Math]::Min($i + $sizePerGroup - 1, $count - 1))]
+        $result += ,$group
+    }
+
+    return $result
+}
+
+$devidedSubModules = Split-List -subModules $subModules -maxParallelJobs $MaxParallelJobs
+
+Write-Host "Total matched devides: $($devidedSubModules.Count)"
+
+$index = 0
+foreach ($subModules in $devidedSubModules) {
+    Write-Host "Outer Group ${index}:"
+    $subIndex = 0
+    foreach ($subModule in $subModules) {
+        Write-Host "Inner Group ${subIndex}: $($subModule -join ',')"
+        $subIndex++
+    }
+
+    $moduleNames = $subModules | ForEach-Object { $_[0] }
+
+    $MatrixStr="$MatrixStr,'" + $($index + 1) + "-" +  $($subModules.Count) + "':{'Target':$($moduleNames -join ',')}"
+
+    $index++
+}
+
+$MatrixStr=$MatrixStr.Substring(1)
+Write-Host "##vso[task.setVariable variable=Targets;isOutput=true]{$MatrixStr}"

.azure-pipelines/batch-generation.yml

@@ -0,0 +1,62 @@
+variables:
+
+  BuildTimeoutInMinutes: 120
+  AnalysisTimeoutInMinutes: 120
+  MaxParallelJobs: 3
+
+trigger: none
+
+jobs:
+- job: prepare
+  displayName: Generate Matrix
+  pool: pool-ubuntu-2004
+  steps:
+  - checkout: self
+    persistCredentials: true
+    clean: true
+    ref: refs/heads/main
+
+  - task: PowerShell@2
+    displayName: 'Create Batch Generation Branch'
+    inputs:
+      targetType: inline
+      script: |
+        $newBranch = "batch-generation/branch-$(Build.BuildId)"
+        git checkout -b $newBranch
+        git push origin $newBranch
+
+  - task: PowerShell@2
+    name: mtrx
+    displayName: 'Generate Matrix'
+    inputs:
+      targetType: inline
+      script: |
+        $prepareModulesPath = Join-Path "$(Build.SourcesDirectory)" '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'prepare.ps1'
+        & $prepareModulesPath -RepoRoot "$(Build.SourcesDirectory)" -MaxParallelJobs "${{ variables.MaxParallelJobs }}"
+  
+
+- job: build
+  displayName: "Build:"
+  dependsOn: prepare
+  timeoutInMinutes: 120
+  pool: pool-windows-2019
+  strategy:
+    matrix: $[ dependencies.prepare.outputs['mtrx.Targets'] ]
+    maxParallel: 15
+
+  steps:
+  - checkout: self
+    persistCredentials: true
+    clean: true
+    ref: "batch-generation/branch-$(Build.BuildId)"
+
+  - task: PowerShell@2
+    name: build
+    displayName: 'Build Targets'
+    inputs:
+      targetType: inline
+      script: |
+        Write-Host "Building targets: ${Target}"
+        $buildModulesPath = Join-Path "$(Build.SourcesDirectory)" 'tools' 'BuildScripts' 'BuildModules.ps1'
+        & $buildModulesPath -TargetModule $($Target -split ',')
+  

.azure-pipelines/test.ps1

@@ -0,0 +1,9 @@
+
+$RepoRoot = Get-Location
+$buildProjPath = Join-Path $RepoRoot 'build.proj'
+
+$filesChangedOutputPath = Join-Path $RepoRoot 'artifacts' 'FilesChanged.txt'
+$subTasksFilePath = Join-Path $RepoRoot 'artifacts' 'SubTasksFile.txt'
+$IsSecurityCheck = $null
+
+dotnet msbuild $buildProjPath /t:FilterBuild "/p:FilesChangedOutputPath=$FilesChangedOutputPath;SubTasksFilePath=$SubTasksFilePath;IsSecurityCheck=$IsSecurityCheck"