Merge pull request #190 from mauriciopoppe/integration-tests-k

Scripts to run the integration tests with a real kubernetes VM

Author: k8s-ci-robot

integrationtests/disk_v1_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/kubernetes-csi/csi-proxy/client/api/disk/v1"
+	v1 "github.com/kubernetes-csi/csi-proxy/client/api/disk/v1"
 	diskv1client "github.com/kubernetes-csi/csi-proxy/client/groups/disk/v1"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -51,19 +51,20 @@ func v1DiskTests(t *testing.T) {
 			t.Errorf("Expected to get at least one diskIDs, instead got diskIDsResponse.DiskIDs=%+v", diskIDsMap)
 		}
 
+		// some disks may have the field Page83, if it's a GCE Persistent disk
+		// it'll have a nonempty SerialNumber
 		// first disk is the VM disk (other disks might be VHD)
-		diskNumber := 0
-		diskIDs, found := diskIDsMap[uint32(diskNumber)]
-		if !found {
-			t.Errorf("Cannot find Disk %d", diskNumber)
-		}
-		page83 := diskIDs.Page83
-		if page83 == "" {
-			t.Errorf("page83 field of diskNumber=%d should be defined, instead got diskIDs=%v", diskNumber, diskIDs)
-		}
-		serialNumber := diskIDs.SerialNumber
-		if serialNumber == "" {
-			t.Errorf("serialNumber field of diskNumber=%d should be defined, instead got diskIDs=%v", diskNumber, diskIDs)
+		for diskNumber, diskIDs := range diskIDsMap {
+			if len(diskIDs.SerialNumber) > 0 {
+				// the nvme disks don't have a Page83 number
+				if strings.HasPrefix(diskIDs.SerialNumber, "nvme") {
+					continue
+				}
+				page83 := diskIDs.Page83
+				if page83 == "" {
+					t.Errorf("page83 field of diskNumber=%d should be defined, instead got diskIDs=%v", diskNumber, diskIDs)
+				}
+			}
 		}
 
 		listDiskLocationsRequest := &v1.ListDiskLocationsRequest{}

integrationtests/disk_v1beta1_test.go

@@ -47,21 +47,6 @@ func v1beta1DiskTests(t *testing.T) {
 			t.Errorf("Expected to get at least one diskIDs, instead got diskIDsResponse.DiskIDs=%+v", diskIDsMap)
 		}
 
-		// first disk is the VM disk (other disks might be VHD)
-		diskNumber := 0
-		diskIDs, found := diskIDsMap[strconv.FormatUint(uint64(diskNumber), 10)]
-		if !found {
-			t.Errorf("Cannot find Disk %d", diskNumber)
-		}
-		page83 := diskIDs.Identifiers["page83"]
-		if page83 == "" {
-			t.Errorf("page83 field of diskNumber=%d should be defined, instead got diskIDs=%v", diskNumber, diskIDs)
-		}
-		serialNumber := diskIDs.Identifiers["serialNumber"]
-		if serialNumber == "" {
-			t.Errorf("serialNumber field of diskNumber=%d should be defined, instead got diskIDs=%v", diskNumber, diskIDs)
-		}
-
 		listDiskLocationsRequest := &v1beta1.ListDiskLocationsRequest{}
 		listDiskLocationsResponse, err := client.ListDiskLocations(context.TODO(), listDiskLocationsRequest)
 		require.Nil(t, err)

integrationtests/disk_v1beta2_test.go

@@ -48,21 +48,6 @@ func v1beta2DiskTests(t *testing.T) {
 			t.Errorf("Expected to get at least one diskIDs, instead got diskIDsResponse.DiskIDs=%+v", diskIDsMap)
 		}
 
-		// first disk is the VM disk (other disks might be VHD)
-		diskNumber := 0
-		diskIDs, found := diskIDsMap[strconv.FormatUint(uint64(diskNumber), 10)]
-		if !found {
-			t.Errorf("Cannot find Disk %d", diskNumber)
-		}
-		page83 := diskIDs.Identifiers["page83"]
-		if page83 == "" {
-			t.Errorf("page83 field of diskNumber=%d should be defined, instead got diskIDs=%v", diskNumber, diskIDs)
-		}
-		serialNumber := diskIDs.Identifiers["serialNumber"]
-		if serialNumber == "" {
-			t.Errorf("serialNumber field of diskNumber=%d should be defined, instead got diskIDs=%v", diskNumber, diskIDs)
-		}
-
 		listDiskLocationsRequest := &v1beta2.ListDiskLocationsRequest{}
 		listDiskLocationsResponse, err := client.ListDiskLocations(context.TODO(), listDiskLocationsRequest)
 		require.Nil(t, err)

integrationtests/disk_v1beta3_test.go

@@ -51,19 +51,20 @@ func v1beta3DiskTests(t *testing.T) {
 			t.Errorf("Expected to get at least one diskIDs, instead got diskIDsResponse.DiskIDs=%+v", diskIDsMap)
 		}
 
+		// some disks may have the field Page83, if it's a GCE Persistent disk
+		// it'll have a nonempty SerialNumber
 		// first disk is the VM disk (other disks might be VHD)
-		diskNumber := 0
-		diskIDs, found := diskIDsMap[uint32(diskNumber)]
-		if !found {
-			t.Errorf("Cannot find Disk %d", diskNumber)
-		}
-		page83 := diskIDs.Page83
-		if page83 == "" {
-			t.Errorf("page83 field of diskNumber=%d should be defined, instead got diskIDs=%v", diskNumber, diskIDs)
-		}
-		serialNumber := diskIDs.SerialNumber
-		if serialNumber == "" {
-			t.Errorf("serialNumber field of diskNumber=%d should be defined, instead got diskIDs=%v", diskNumber, diskIDs)
+		for diskNumber, diskIDs := range diskIDsMap {
+			if len(diskIDs.SerialNumber) > 0 {
+				// the nvme disks don't have a Page83 number
+				if strings.HasPrefix(diskIDs.SerialNumber, "nvme") {
+					continue
+				}
+				page83 := diskIDs.Page83
+				if page83 == "" {
+					t.Errorf("page83 field of diskNumber=%d should be defined, instead got diskIDs=%v", diskNumber, diskIDs)
+				}
+			}
 		}
 
 		listDiskLocationsRequest := &v1beta3.ListDiskLocationsRequest{}

integrationtests/utils.go

@@ -156,7 +156,7 @@ func shouldRunIscsiTests() bool {
 }
 
 func runPowershellCmd(t *testing.T, command string) (string, error) {
-	cmd := exec.Command("powershell", "/c", command)
+	cmd := exec.Command("powershell", "/c", fmt.Sprintf("& { $global:ProgressPreference = 'SilentlyContinue'; %s }", command))
 	t.Logf("Executing command: %q", cmd.String())
 	result, err := cmd.CombinedOutput()
 	return string(result), err
@@ -205,7 +205,7 @@ func diskInit(t *testing.T) (*VirtualHardDisk, func()) {
 	s1 := rand.NewSource(time.Now().UTC().UnixNano())
 	r1 := rand.New(s1)
 
-	testId := r1.Intn(1000)
+	testId := r1.Intn(10000000)
 	testPluginPath := fmt.Sprintf("C:\\var\\lib\\kubelet\\plugins\\testplugin-%d.csi.io\\", testId)
 	mountPath := fmt.Sprintf("%smount-%d", testPluginPath, testId)
 	vhdxPath := fmt.Sprintf("%sdisk-%d.vhdx", testPluginPath, testId)

scripts/bump-version.sh

@@ -1,4 +1,4 @@
-#/bin/bash
+#!/bin/bash
 
 # bump-version.sh
 # Bumps an API version from an old version to a new one

scripts/run-integration.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#
+# Runs the integration tests
+#
+# Requirements:
+# - a kubernetes cluster with a Windows nodepool
+#
+# Steps:
+# - cross compile the csi-proxy binary and the integration tests
+# - copy to the VM using scp
+# - restart the CSI Proxy binary process with a helper powershell script
+# - run the integration tests
+
+set -euxo pipefail
+
+pkgdir=${GOPATH}/src/github.com/kubernetes-csi/csi-proxy
+source $pkgdir/scripts/utils.sh
+
+main() {
+  compile_csi_proxy
+  compile_csi_proxy_integration_tests
+  sync_csi_proxy
+  sync_csi_proxy_integration_tests
+  sync_powershell_utils
+  restart_csi_proxy
+  run_csi_proxy_integration_tests
+}
+
+main

scripts/sync-csi-proxy.ps1

@@ -1,4 +1,5 @@
-#/bin/bash
+#!/bin/bash
+
 #
 # Installs CSI Proxy in a kubernetes node
 #
@@ -10,25 +11,16 @@
 # - copy to the VM using scp
 # - restart the CSI Proxy binary process with a helper powershell script
 
-set -ex
+set -euxo pipefail
 
-script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-csi_proxy_bin_path="$script_dir/../bin/csi-proxy.exe"
-sync_script_ps_path="$script_dir/sync-csi-proxy.ps1"
+pkgdir=${GOPATH}/src/github.com/kubernetes-csi/csi-proxy
+source $pkgdir/scripts/utils.sh
 
 main() {
-  echo "Compiling CSI Proxy"
-  make build
-
-  local windows_node=$(kubectl get nodes -l kubernetes.io/os=windows -o jsonpath='{.items[*].metadata.name}')
-  gcloud compute instances describe $windows_node > /dev/null
-
-  echo "Sync the csi-proxy.exe binary"
-  local current_account=$(gcloud config list account --format "value(core.account)" | sed -r 's/@\S+//g')
-  gcloud compute scp $csi_proxy_bin_path $sync_script_ps_path $windows_node:"C:\\Users\\${current_account}"
-
-  echo "Restart csi-proxy service"
-  gcloud compute ssh $windows_node --command="powershell .\sync-csi-proxy.ps1"
+  compile_csi_proxy
+  sync_csi_proxy
+  sync_powershell_utils
+  restart_csi_proxy
 }
 
 main

scripts/sync-csi-proxy.sh

@@ -0,0 +1,41 @@
+function Restart-CSIProxy {
+  # stop the csiproxy service
+  Get-Process csi-proxy | Stop-Process -Force
+  sc.exe stop csiproxy
+  Start-Sleep -Seconds 1;
+  sc.exe delete csiproxy
+  Start-Sleep -Seconds 1;
+
+  # copy the binary from the user directory
+  Copy-Item -Path "C:\Users\$env:UserName\csi-proxy.exe" -Destination "C:\etc\kubernetes\node\bin\csi-proxy.exe"
+
+  # restart the csiproxy service
+  $flags = "-v=5 -windows-service -log_file=C:\etc\kubernetes\logs\csi-proxy.log -logtostderr=false"
+  sc.exe create csiproxy start= "auto" binPath= "C:\etc\kubernetes\node\bin\csi-proxy.exe $flags"
+  sc.exe failure csiproxy reset= 0 actions= restart/10000
+  sc.exe start csiproxy
+
+  Start-Sleep -Seconds 5;
+
+  Write-Output "Checking the status of csi-proxy"
+  sc.exe query csiproxy
+  [System.IO.Directory]::GetFiles("\\.\\pipe\\")
+
+  Write-Output "Get logs"
+  Get-Content C:\etc\kubernetes\logs\csi-proxy.log -Tail 20
+}
+
+function Run-CSIProxyIntegrationTests {
+  param (
+    [string]$test_args = ""
+  )
+
+  $ErrorActionPreference = "Stop";
+  Write-Output "Running integration tests with test_args=$test_args";
+  $flags = $test_args.Split(" ");
+  .\integrationtests.test.exe @flags
+
+  if (-not $?) {
+    throw "failed to run with exit code=$?"
+  }
+}

scripts/utils.psm1

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Importing this library shouldn't have side effects
+
+pkgdir=${GOPATH}/src/github.com/kubernetes-csi/csi-proxy
+
+# current_account is the current user
+# in CI, the value is `prow`
+current_account=$USER
+# windows_node is the id of the GCE Windows instance
+windows_node=$(kubectl get nodes -l kubernetes.io/os=windows -o jsonpath='{.items[*].metadata.name}')
+
+# set the default zone for the gcloud sdk
+if ! [ -z "${GCP_ZONE:-}" ]; then
+  export CLOUDSDK_COMPUTE_ZONE="$GCP_ZONE"
+fi
+
+sync_file_to_vm() {
+  gcloud compute scp $@ $windows_node:"C:\\Users\\${current_account}"
+}
+
+compile_csi_proxy() {
+  echo "Compiling CSI Proxy"
+  make -C $pkgdir build
+}
+
+compile_csi_proxy_integration_tests() {
+  echo "Compiling CSI Proxy integration tests"
+  GOOS=windows GOARCH=amd64 go test -c $pkgdir/integrationtests -o $pkgdir/bin/integrationtests.test.exe
+}
+
+sync_csi_proxy() {
+  echo "Sync the csi-proxy.exe binary"
+  local csi_proxy_bin_path="$pkgdir/bin/csi-proxy.exe"
+  sync_file_to_vm $csi_proxy_bin_path
+}
+
+sync_csi_proxy_integration_tests() {
+  echo "Sync the integrationtests.exe binary"
+  local integration_bin_path="$pkgdir/bin/integrationtests.test.exe"
+  sync_file_to_vm $integration_bin_path
+}
+
+sync_powershell_utils() {
+  local utils_psm1="$pkgdir/scripts/utils.psm1"
+  sync_file_to_vm $utils_psm1
+}
+
+restart_csi_proxy() {
+  echo "Restart csi-proxy service"
+  gcloud compute ssh $windows_node --command='powershell -c "& { $ErrorActionPreference = \"Stop\"; Import-Module (Resolve-Path(\"utils.psm1\")); Restart-CSIProxy; }"'
+}
+
+run_csi_proxy_integration_tests() {
+  echo "Run integration tests"
+  local ps1=$(cat << 'EOF'
+  "& {
+    $ErrorActionPreference = \"Stop\";
+    Import-Module (Resolve-Path(\"utils.psm1\"));
+    Run-CSIProxyIntegrationTests -test_args \"--test.v --test.run TestAPIGroups\";
+    Run-CSIProxyIntegrationTests -test_args \"--test.v --test.run TestFilesystemAPIGroup\";
+    Run-CSIProxyIntegrationTests -test_args \"--test.v --test.run TestDiskAPIGroup\";
+    Run-CSIProxyIntegrationTests -test_args \"--test.v --test.run TestVolumeAPIs\";
+    Run-CSIProxyIntegrationTests -test_args \"--test.v --test.run TestSmbAPIGroup\";
+  }"
+EOF
+);
+
+  gcloud compute ssh $windows_node --command="powershell -c $(echo $ps1 | tr '\n' ' ')"
+}