testworkflow.go

// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package imagetest is a library for the orchestration components of CIT. It
// executes testworkflows (generated by test suites), collates the results into
// junit xml.
package imagetest

import (
	"context"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"path"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"cloud.google.com/go/storage"
	"github.com/GoogleCloudPlatform/cloud-image-tests/cleanerupper"
	"github.com/GoogleCloudPlatform/cloud-image-tests/utils"
	daisy "github.com/GoogleCloudPlatform/compute-daisy"
	daisycompute "github.com/GoogleCloudPlatform/compute-daisy/compute"
	"github.com/jstemmer/go-junit-report/v2/junit"
	computeBeta "google.golang.org/api/compute/v0.beta"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/iterator"
)

var (
	client *storage.Client
)

const (
	// PdStandard disk type string
	PdStandard = "pd-standard"
	// PdSsd disk type string
	PdSsd = "pd-ssd"
	// PdBalanced disk type string
	PdBalanced = "pd-balanced"
	// PdExtreme disk type string
	PdExtreme = "pd-extreme"
	// HyperdiskExtreme disk type string
	HyperdiskExtreme = "hyperdisk-extreme"
	// HyperdiskThroughput disk type string
	HyperdiskThroughput = "hyperdisk-throughput"
	// HyperdiskBalanced disk type string
	HyperdiskBalanced = "hyperdisk-balanced"
	// LocalSsd disk type string
	LocalSsd = "local-ssd"

	// guestAttributesKey is the metadata key used to enable guest attributes.
	guestAttributesKey = "enable-guest-attributes"

	testWrapperPath        = "/wrapper"
	testWrapperPathWindows = "/wrapp"
)

// TestWorkflowOpts is an options struct for the NewTestWorkflow function.
type TestWorkflowOpts struct {
	// Client is the client used to call the compute service
	Client daisycompute.Client
	// ComputeEndpointOverride is an alternate compute endpoint to send requests to compute service to.
	ComputeEndpointOverride string
	// Name is the name of the TestWorkflow
	Name string
	// Image is an image partial URL, or an image family partial URL
	Image string
	// The daisy step timeout
	Timeout string
	// Project is the project used to look up images and set up the test workflow.
	Project string
	// Zone is the zone used to run the test workflow.
	Zone string
	// Zones is the list of zones to try in case of stockout or quota issues.
	Zones []string
	// ExcludeFilter is a filter used to exclude individual test cases within a test suite. Can break test suites if used incorrectly, be careful.
	ExcludeFilter string
	// X86Shape is the default shape for x86 images being tested.
	X86Shape string
	// ARM64Shape is the default shape for ARM images being tested.
	ARM64Shape string
	// UseReservations is whether to consume reservations to create VMs. If true but ReservationURLs is empty, ANY_RESERVATION will be used.
	UseReservations bool
	// ReservationURLs is a list of specific reservation URLs to consume.
	ReservationURLs []string
	// ReservationBound is whether to set provisioningModel=RESERVATION_BOUND when using specific reservations. Defaults to true for backwards compatibility with calendar/dense reservations.
	ReservationBound bool
	// AcceleratorType is the accelerator type to be used for accelerator tests, which use GPUs.
	AcceleratorType string
	// AcceleratorCount is the number of accelerators to be used for accelerator tests.
	AcceleratorCount int64
	// ArgZoneOverride is whether to override the zone from the command line. If
	// true, the zone from the command line will be enforced if the test suite
	// does specify a zone. If false, the hardcoded zone will be used.
	ArgZoneOverride bool
	// ExternalIP is the external IP to use for VMs (ephemeral or none).
	ExternalIP string
	// Network is the network to use for VMs. This is used for tests that do not configure custom network interfaces.
	Network string
	// Subnet is the subnet to use for VMs. This is used for tests that do not configure custom network interfaces.
	Subnet string
}

// TestWorkflow defines a test workflow which creates at least one test VM.
type TestWorkflow struct {
	Name string
	// Client is a shared client for the compute service.
	Client daisycompute.Client
	// Image is the image under test
	Image *compute.Image
	// ImageBeta is the image under test using Beta API
	ImageBeta *computeBeta.Image
	// ImageURL will be the partial URL of a GCE image.
	ImageURL string
	// MachineType is the machine type to be used for the test. This can be overridden by individual test suites.
	MachineType *compute.MachineType
	Project     *compute.Project
	Zone        *compute.Zone
	// GCSPath is the destination for workflow outputs in gs://[...] form.
	GCSPath           string
	skipped           bool
	skippedMessage    string
	testExcludeFilter string
	wf                *daisy.Workflow
	// Global counter for all daisy steps on all VMs. This is an interim solution in order to prevent step-name collisions.
	counter int
	// Does this test require exclusive project
	lockProject bool
	// ReservationAffinity is the reservation affinity used for VM creation.
	ReservationAffinity *compute.ReservationAffinity
	// ReservationAffinityBeta is the reservation affinity used for VM creation with the beta API.
	ReservationAffinityBeta *computeBeta.ReservationAffinity
	// ReservationBound is whether to set provisioningModel=RESERVATION_BOUND when using specific reservations.
	ReservationBound bool
	// AcceleratorType is the accelerator type to be used for accelerator tests which use GPUs.
	AcceleratorType string
	// AcceleratorCount is the number of accelerators to be used for accelerator tests.
	AcceleratorCount int64
	// CreatesLoadBalancers indicates that this test suite creates regional load
	// balancer resources (forwarding rules, backend services, URL maps, target
	// HTTP proxies, NEGs, health checks) at runtime. Suites that set this true
	// opt into the post-test LB cleanup pass; suites that leave it false skip
	// the call entirely. Default false because only the loadbalancer suite
	// creates LB resources today.
	CreatesLoadBalancers bool
	// argZoneOverride is whether to override the zone from the command line. If
	// true, the zone from the command line will be enforced if the test suite
	// does specify a zone. If false, the hardcoded zone will be used.
	argZoneOverride bool
	// opts is the options struct for the TestWorkflow. Storing the original options
	// is necessary so we can copy them and construct a clean recreation of the
	// stateful, zone-bound Daisy workflows during stockout retries in a new zone.
	opts *TestWorkflowOpts
	// SetupFunc is an optional function that is called once at the beginning of the test workflow.
	SetupFunc func(*TestWorkflow) error
}

func (t *TestWorkflow) setInstanceTestMetadata(instance *daisy.Instance, suffix string) {
	name := instance.Name

	instance.Metadata["_test_vmname"] = name
	instance.Metadata["_test_package_url"] = "${SOURCESPATH}/testpackage"
	instance.Metadata["_test_properties_url"] = fmt.Sprintf("${OUTSPATH}/properties/%s.txt", instance.Name)
	instance.Metadata["_test_package_name"] = fmt.Sprintf("image_test%s", suffix)
	instance.Metadata["_test_results_url"] = fmt.Sprintf("${OUTSPATH}/%s.txt", name)
	instance.Metadata["_test_suite_name"] = getTestSuiteName(t)
	instance.Metadata["_compute_endpoint"] = t.wf.ComputeEndpoint
	instance.Metadata["_cit_timeout"] = t.wf.DefaultTimeout
	instance.Metadata["_exclude_discrete_tests"] = t.testExcludeFilter
	instance.Metadata["enable-guest-attributes"] = "TRUE" // enable guest attributes by default.
}

// addNewVMStep adds an entirely new addVM step, separate from the step used and
// modified by the `appendCreateVMStep` function.
func (t *TestWorkflow) addNewVMStep(disks []*compute.Disk, instanceParams *daisy.Instance) (*daisy.Step, *daisy.Instance, error) {
	stepSuffix := fmt.Sprintf("%d", t.counter)
	if len(disks) == 0 || disks[0].Name == "" {
		return nil, nil, fmt.Errorf("failed to create VM from empty boot disk")
	}
	// The boot disk is the first disk, and the VM name comes from that
	name := disks[0].Name
	if strings.Contains(name, "-") {
		return nil, nil, fmt.Errorf("dashes are disallowed in testworkflow vm names: %s", name)
	}

	isWindows := utils.HasFeature(t.Image, "WINDOWS")
	var suffix string
	if isWindows {
		suffix = ".exe"
	}

	instance := instanceParams
	if instance == nil {
		instance = &daisy.Instance{}
	}

	instance.StartupScript = fmt.Sprintf("wrapper%s", suffix)
	instance.Name = name
	instance.Scopes = append(instance.Scopes, "https://www.googleapis.com/auth/devstorage.read_write")

	for _, disk := range disks {
		currentDisk := &compute.AttachedDisk{Source: disk.Name, AutoDelete: true}
		instance.Disks = append(instance.Disks, currentDisk)
	}

	if instance.Metadata == nil {
		instance.Metadata = make(map[string]string)
	}

	t.setInstanceTestMetadata(instance, suffix)
	t.skipWindowsStagingKMS(isWindows, instance)

	createInstances := &daisy.CreateInstances{}
	createInstances.Instances = append(createInstances.Instances, instance)

	createVMsStepName := fmt.Sprintf("create-vms-%s", stepSuffix)
	createVMsStep, err := t.wf.NewStep(createVMsStepName)
	if err != nil {
		return nil, nil, err
	}
	createVMsStep.CreateInstances = createInstances
	return createVMsStep, instance, nil
}

func (t *TestWorkflow) matchesWorkflowMachineType(instanceMachineType string) bool {
	return instanceMachineType == "" || t == nil || t.MachineType == nil || t.MachineType.Name == "" || instanceMachineType == t.MachineType.Name
}

func (t *TestWorkflow) appendCreateVMStep(disks []*compute.Disk, instanceParams *daisy.Instance) (*daisy.Step, *daisy.Instance, error) {
	if len(disks) == 0 || disks[0].Name == "" {
		return nil, nil, fmt.Errorf("failed to create VM from empty boot disk")
	}
	// The boot disk is the first disk, and the VM name comes from that
	name := disks[0].Name
	if strings.Contains(name, "-") {
		return nil, nil, fmt.Errorf("dashes are disallowed in testworkflow vm names: %s", name)
	}

	isWindows := utils.HasFeature(t.Image, "WINDOWS")
	var suffix string
	if isWindows {
		suffix = ".exe"
	}

	instance := instanceParams
	if instance == nil {
		instance = &daisy.Instance{}
	}
	if policy := MachineMaintenancePolicy(t.MachineType.Name); policy == "TERMINATE" {
		if instance.Scheduling == nil {
			instance.Scheduling = &compute.Scheduling{}
		}
		instance.Scheduling.OnHostMaintenance = policy
		log.Printf("Setting onHostMaintenance to %s for VM %s (machine type %s)", policy, name, t.MachineType.Name)
	}
	instance.StartupScript = fmt.Sprintf("wrapper%s", suffix)
	instance.Name = name
	instance.Scopes = append(instance.Scopes, "https://www.googleapis.com/auth/devstorage.read_write")
	if t.matchesWorkflowMachineType(instance.MachineType) {
		instance.ReservationAffinity = t.ReservationAffinity
		if t.ReservationAffinity != nil && t.ReservationAffinity.ConsumeReservationType == "SPECIFIC_RESERVATION" && t.ReservationBound {
			if instance.Scheduling == nil {
				instance.Scheduling = &compute.Scheduling{}
			}
			instance.Scheduling.ProvisioningModel = "RESERVATION_BOUND"
		}
	}

	for _, disk := range disks {
		currentDisk := &compute.AttachedDisk{Source: disk.Name, AutoDelete: true}
		currentDisk.AutoDelete = true
		instance.Disks = append(instance.Disks, currentDisk)
	}

	if instance.Metadata == nil {
		instance.Metadata = make(map[string]string)
	}

	t.setInstanceTestMetadata(instance, suffix)
	t.skipWindowsStagingKMS(isWindows, instance)

	createInstances := &daisy.CreateInstances{}
	createInstances.Instances = append(createInstances.Instances, instance)

	createVMStep, ok := t.wf.Steps[createVMsStepName]
	if ok {
		// append to existing step.
		createVMStep.CreateInstances.Instances = append(createVMStep.CreateInstances.Instances, instance)
	} else {
		var err error
		createVMStep, err = t.wf.NewStep(createVMsStepName)
		if err != nil {
			return nil, nil, err
		}
		createVMStep.CreateInstances = createInstances
	}

	return createVMStep, instance, nil
}

func (t *TestWorkflow) appendCreateVMStepBeta(disks []*compute.Disk, instance *daisy.InstanceBeta) (*daisy.Step, *daisy.InstanceBeta, error) {
	if len(disks) == 0 || disks[0].Name == "" {
		return nil, nil, fmt.Errorf("failed to create VM from empty boot disk")
	}
	// The boot disk is the first disk, and the VM name comes from that
	name := disks[0].Name

	var suffix string
	if utils.HasFeature(t.Image, "WINDOWS") {
		suffix = ".exe"
	}

	if instance == nil {
		instance = &daisy.InstanceBeta{}
	}
	if policy := MachineMaintenancePolicy(t.MachineType.Name); policy == "TERMINATE" {
		if instance.Scheduling == nil {
			instance.Scheduling = &computeBeta.Scheduling{}
		}
		instance.Scheduling.OnHostMaintenance = policy
		log.Printf("Setting onHostMaintenance to %s for VM %s (machine type %s)", policy, name, t.MachineType.Name)
	}
	instance.StartupScript = fmt.Sprintf("wrapper%s", suffix)
	instance.Name = name
	instance.Scopes = append(instance.Scopes, "https://www.googleapis.com/auth/devstorage.read_write")
	if t.matchesWorkflowMachineType(instance.MachineType) {
		instance.ReservationAffinity = t.ReservationAffinityBeta
		if t.ReservationAffinityBeta != nil && t.ReservationAffinityBeta.ConsumeReservationType == "SPECIFIC_RESERVATION" && t.ReservationBound {
			if instance.Scheduling == nil {
				instance.Scheduling = &computeBeta.Scheduling{}
			}
			instance.Scheduling.ProvisioningModel = "RESERVATION_BOUND"
		}
	}

	for _, disk := range disks {
		instance.Disks = append(instance.Disks, &computeBeta.AttachedDisk{Source: disk.Name, AutoDelete: true})
	}

	if instance.Metadata == nil {
		instance.Metadata = make(map[string]string)
	}

	instance.Metadata["_test_vmname"] = name
	instance.Metadata["_test_package_url"] = "${SOURCESPATH}/testpackage"
	instance.Metadata["_test_results_url"] = fmt.Sprintf("${OUTSPATH}/%s.txt", name)
	instance.Metadata["_test_properties_url"] = fmt.Sprintf("${OUTSPATH}/properties/%s.txt", name)
	instance.Metadata["_test_suite_name"] = getTestSuiteName(t)
	instance.Metadata["_test_package_name"] = fmt.Sprintf("image_test%s", suffix)
	instance.Metadata["_compute_endpoint"] = t.wf.ComputeEndpoint
	instance.Metadata["_exclude_discrete_tests"] = t.testExcludeFilter
	instance.Metadata["enable-guest-attributes"] = "TRUE" // enable guest attributes by default.

	createInstances := &daisy.CreateInstances{}
	createInstances.InstancesBeta = append(createInstances.InstancesBeta, instance)

	createVMStep, ok := t.wf.Steps[createVMsStepName]
	if ok {
		// append to existing step.
		createVMStep.CreateInstances.InstancesBeta = append(createVMStep.CreateInstances.InstancesBeta, instance)
	} else {
		var err error
		createVMStep, err = t.wf.NewStep(createVMsStepName)
		if err != nil {
			return nil, nil, err
		}
		createVMStep.CreateInstances = createInstances
	}
	instance.Metadata["_cit_timeout"] = t.wf.DefaultTimeout

	return createVMStep, instance, nil
}

// appendCreateDisksStep should be called for creating the boot disk, or first disk in a VM.
func (t *TestWorkflow) appendCreateDisksStep(diskParams *compute.Disk) (*daisy.Step, error) {
	if diskParams == nil || diskParams.Name == "" {
		return nil, fmt.Errorf("failed to create disk with empty parameters")
	}
	bootdisk := &daisy.Disk{}
	bootdisk.Name = diskParams.Name
	bootdisk.SourceImage = t.ImageURL
	bootdisk.Type = diskParams.Type
	bootdisk.Zone = diskParams.Zone
	bootdisk.SizeGb = strconv.FormatInt(diskParams.SizeGb, 10)

	createDisks := &daisy.CreateDisks{bootdisk}

	createDisksStep, ok := t.wf.Steps[createDisksStepName]
	if ok {
		// append to existing step.
		*createDisksStep.CreateDisks = append(*createDisksStep.CreateDisks, bootdisk)
	} else {
		var err error
		createDisksStep, err = t.wf.NewStep(createDisksStepName)
		if err != nil {
			return nil, err
		}
		createDisksStep.CreateDisks = createDisks
	}

	return createDisksStep, nil
}

// appendCreateMountDisksStep should be called for any disk which is not the vm boot disk.
func (t *TestWorkflow) appendCreateMountDisksStep(diskParams *compute.Disk) (*daisy.Step, error) {
	if diskParams == nil || diskParams.Name == "" {
		return nil, fmt.Errorf("failed to create disk with empty parameters")
	}
	mountdisk := &daisy.Disk{}
	mountdisk.Name = diskParams.Name
	mountdisk.Type = diskParams.Type
	mountdisk.Zone = diskParams.Zone
	if diskParams.SizeGb == 0 {
		return nil, fmt.Errorf("failed to create mount disk with no SizeGb parameter")
	}
	mountdisk.SizeGb = strconv.FormatInt(diskParams.SizeGb, 10)

	createDisks := &daisy.CreateDisks{mountdisk}

	createDisksStep, ok := t.wf.Steps[createDisksStepName]
	if ok {
		// append to existing step.
		*createDisksStep.CreateDisks = append(*createDisksStep.CreateDisks, mountdisk)
	} else {
		var err error
		createDisksStep, err = t.wf.NewStep(createDisksStepName)
		if err != nil {
			return nil, err
		}
		createDisksStep.CreateDisks = createDisks
	}

	return createDisksStep, nil
}

func (t *TestWorkflow) appendCreateImageStep(stepname string, image *compute.Image) (*daisy.Step, error) {
	createImages := &daisy.Image{}
	createImages.Name = image.Name
	createImages.SourceDisk = image.SourceDisk
	createImagesStep, ok := t.wf.Steps[stepname]
	if ok {
		// append to existing step.
		createImagesStep.CreateImages.Images = append(createImagesStep.CreateImages.Images, createImages)
	} else {
		var err error
		createImagesStep, err = t.wf.NewStep(stepname)
		if err != nil {
			return nil, err
		}
		createImagesStep.CreateImages.Images = []*daisy.Image{createImages}
	}
	return createImagesStep, nil
}

// Detaches all the specified disks from the specified VM.
func (t *TestWorkflow) appendDetachDiskStep(stepname, vmname string, disknames []string) (*daisy.Step, error) {
	disks := &daisy.DetachDisks{}
	for _, diskname := range disknames {
		detachDisk := &daisy.DetachDisk{}
		detachDisk.Instance = vmname
		detachDisk.DeviceName = diskname

		*disks = append(*disks, detachDisk)
	}

	detachDisksStep, err := t.wf.NewStep(stepname)
	if err != nil {
		return nil, err
	}
	detachDisksStep.DetachDisks = disks
	return detachDisksStep, nil
}

func (t *TestWorkflow) addWaitStoppedStep(stepname, vmname string) (*daisy.Step, error) {
	instanceSignal := &daisy.InstanceSignal{}
	instanceSignal.Name = vmname
	instanceSignal.Stopped = true

	waitForInstances := &daisy.WaitForInstancesSignal{instanceSignal}
	waitStep, err := t.wf.NewStep("wait-stopped-" + stepname)
	if err != nil {
		return nil, err
	}
	waitStep.WaitForInstancesSignal = waitForInstances

	return waitStep, nil
}

func (t *TestWorkflow) addWaitStep(stepname, vmname string) (*daisy.Step, error) {
	serialOutput := &daisy.SerialOutput{}
	serialOutput.Port = 1
	serialOutput.SuccessMatch = successMatch

	instanceSignal := &daisy.InstanceSignal{}
	instanceSignal.Name = vmname
	instanceSignal.Stopped = false

	guestAttribute := &daisy.GuestAttribute{}
	guestAttribute.Namespace = utils.GuestAttributeTestNamespace
	guestAttribute.KeyName = utils.GuestAttributeTestKey

	instanceSignal.SerialOutput = serialOutput
	instanceSignal.GuestAttribute = guestAttribute
	instanceSignal.Interval = "8s"

	waitForInstances := &daisy.WaitForInstancesSignal{instanceSignal}

	waitStep, err := t.wf.NewStep("wait-" + stepname)
	if err != nil {
		return nil, err
	}
	waitStep.WaitForInstancesSignal = waitForInstances

	return waitStep, nil
}

// after guest attributes for instance wait step matching are implemented, this step will wait for a different guest attribute key than addWaitStep
func (t *TestWorkflow) addWaitRebootGAStep(stepname, vmname string) (*daisy.Step, error) {
	serialOutput := &daisy.SerialOutput{}
	serialOutput.Port = 1
	serialOutput.SuccessMatch = successMatch

	instanceSignal := &daisy.InstanceSignal{}
	instanceSignal.Name = vmname
	instanceSignal.Stopped = false

	guestAttribute := &daisy.GuestAttribute{}
	guestAttribute.Namespace = utils.GuestAttributeTestNamespace
	// specifically wait for a different guest attribute if this is the
	// first boot before a reboot, and we want test results from a reboot.
	guestAttribute.KeyName = utils.FirstBootGAKey

	instanceSignal.SerialOutput = serialOutput
	instanceSignal.GuestAttribute = guestAttribute
	instanceSignal.Interval = "8s"

	waitForInstances := &daisy.WaitForInstancesSignal{instanceSignal}

	waitStep, err := t.wf.NewStep("wait-" + stepname)
	if err != nil {
		return nil, err
	}
	waitStep.WaitForInstancesSignal = waitForInstances

	return waitStep, nil
}

func (t *TestWorkflow) addStopStep(stepname, vmname string) (*daisy.Step, error) {
	stopInstances := &daisy.StopInstances{}
	stopInstances.Instances = append(stopInstances.Instances, vmname)

	stopInstancesStep, err := t.wf.NewStep("stop-" + stepname)
	if err != nil {
		return nil, err
	}
	stopInstancesStep.StopInstances = stopInstances

	return stopInstancesStep, nil
}

func (t *TestWorkflow) addDiskResizeStep(stepname, vmname string, diskSize int) (*daisy.Step, error) {
	resizeDisk := &daisy.ResizeDisk{}
	resizeDisk.DisksResizeRequest.SizeGb = int64(diskSize)
	resizeDisk.Name = vmname
	resizeDiskStepName := "resize-disk-" + stepname
	resizeDiskStep, err := t.wf.NewStep(resizeDiskStepName)
	if err != nil {
		return nil, err
	}
	resizeDiskStep.ResizeDisks = &daisy.ResizeDisks{resizeDisk}

	return resizeDiskStep, nil
}

func (t *TestWorkflow) addStartStep(stepname, vmname string) (*daisy.Step, error) {
	startInstances := &daisy.StartInstances{}
	startInstances.Instances = append(startInstances.Instances, vmname)

	startInstancesStep, err := t.wf.NewStep("start-" + stepname)
	if err != nil {
		return nil, err
	}
	startInstancesStep.StartInstances = startInstances

	return startInstancesStep, nil
}

func (t *TestWorkflow) appendCreateNetworkStep(network *daisy.Network) (*daisy.Step, *daisy.Network, error) {
	createNetworks := &daisy.CreateNetworks{}
	*createNetworks = append(*createNetworks, network)
	createNetworkStep, ok := t.wf.Steps[createNetworkStepName]
	if ok {
		// append to existing step.
		*createNetworkStep.CreateNetworks = append(*createNetworkStep.CreateNetworks, network)
	} else {
		var err error
		createNetworkStep, err = t.wf.NewStep(createNetworkStepName)
		if err != nil {
			return nil, nil, err
		}
		createNetworkStep.CreateNetworks = createNetworks
	}

	return createNetworkStep, network, nil
}

func (t *TestWorkflow) appendCreateSubnetworksStep(subnetwork *daisy.Subnetwork) (*daisy.Step, *daisy.Subnetwork, error) {
	createSubnetworks := &daisy.CreateSubnetworks{}
	*createSubnetworks = append(*createSubnetworks, subnetwork)

	createSubnetworksStep, ok := t.wf.Steps[createSubnetworkStepName]
	if ok {
		// append to existing step.
		*createSubnetworksStep.CreateSubnetworks = append(*createSubnetworksStep.CreateSubnetworks, subnetwork)
	} else {
		var err error
		createSubnetworksStep, err = t.wf.NewStep(createSubnetworkStepName)
		if err != nil {
			return nil, nil, err
		}

		createSubnetworksStep.CreateSubnetworks = createSubnetworks
	}

	return createSubnetworksStep, subnetwork, nil
}

func (t *TestWorkflow) configureNetwork(step *daisy.Step) {
	if t.opts == nil {
		return
	}
	if t.opts.Network == "" && t.opts.Subnet == "" {
		return
	}
	if step.CreateInstances == nil {
		return
	}
	for _, instance := range step.CreateInstances.Instances {
		if instance.Instance.NetworkInterfaces == nil || len(instance.Instance.NetworkInterfaces) == 0 {
			instance.Instance.NetworkInterfaces = []*compute.NetworkInterface{{}}
			if t.opts.Network != "" {
				instance.Instance.NetworkInterfaces[0].Network = t.opts.Network
			}
			if t.opts.Subnet != "" {
				instance.Instance.NetworkInterfaces[0].Subnetwork = t.opts.Subnet
			}
		}
	}
	for _, instance := range step.CreateInstances.InstancesBeta {
		if instance.Instance.NetworkInterfaces == nil || len(instance.Instance.NetworkInterfaces) == 0 {
			instance.Instance.NetworkInterfaces = []*computeBeta.NetworkInterface{{}}
			if t.opts.Network != "" {
				instance.Instance.NetworkInterfaces[0].Network = t.opts.Network
			}
			if t.opts.Subnet != "" {
				instance.Instance.NetworkInterfaces[0].Subnetwork = t.opts.Subnet
			}
		}
	}
}

func (t *TestWorkflow) configureExternalIP(step *daisy.Step) {
	if t.opts == nil {
		return
	}
	if step.CreateInstances == nil {
		return
	}

	if t.opts.ExternalIP == "" {
		return
	}

	// Ensure NetworkInterfaces is initialized for all instances.
	for _, instance := range step.CreateInstances.Instances {
		if instance.Instance.NetworkInterfaces == nil || len(instance.Instance.NetworkInterfaces) == 0 {
			instance.Instance.NetworkInterfaces = []*compute.NetworkInterface{{}}
		}
	}
	for _, instance := range step.CreateInstances.InstancesBeta {
		if instance.Instance.NetworkInterfaces == nil || len(instance.Instance.NetworkInterfaces) == 0 {
			instance.Instance.NetworkInterfaces = []*computeBeta.NetworkInterface{{}}
		}
	}

	if strings.EqualFold(t.opts.ExternalIP, "none") {
		daisy.UpdateInstanceNoExternalIP(step)
	} else {
		// Ephemeral or empty
		for _, instance := range step.CreateInstances.Instances {
			for _, networkInterface := range instance.Instance.NetworkInterfaces {
				if networkInterface.AccessConfigs == nil {
					networkInterface.AccessConfigs = []*compute.AccessConfig{{Type: "ONE_TO_ONE_NAT"}}
				}
				if len(networkInterface.AccessConfigs) > 0 {
					networkInterface.AccessConfigs[0].NatIP = ""
				}
			}
		}
		for _, instance := range step.CreateInstances.InstancesBeta {
			for _, networkInterface := range instance.Instance.NetworkInterfaces {
				if networkInterface.AccessConfigs == nil {
					networkInterface.AccessConfigs = []*computeBeta.AccessConfig{{Type: "ONE_TO_ONE_NAT"}}
				}
				if len(networkInterface.AccessConfigs) > 0 {
					networkInterface.AccessConfigs[0].NatIP = ""
				}
			}
		}
	}
}

func getGCSPrefix(ctx context.Context, storageClient *storage.Client, project, gcsPath string) (string, error) {
	// Set global client.
	client = storageClient

	// If user didn't specify gcsPath, detect or create the bucket.
	if gcsPath == "" {
		bucket, err := daisyBucket(ctx, client, project)
		if err != nil {
			return "", fmt.Errorf("failed to find or create daisy bucket: %v", err)
		}
		gcsPath = fmt.Sprintf("gs://%s", bucket)
	}
	return fmt.Sprintf("%s/%s", strings.TrimSuffix(gcsPath, "/"), time.Now().Format(time.RFC3339)), nil
}

func logGCSDirectoryURL(gsPath string) {
	p := strings.TrimPrefix(gsPath, "gs://")
	link := fmt.Sprintf("https://console.cloud.google.com/storage/browser/%s", p)
	log.Printf("Logs and artifacts can be found at: %s", link)
}

// finalizeWorkflows adds the final necessary data to each workflow for it to
// be able to run, including the final copy-objects step.
func finalizeWorkflows(ctx context.Context, tests []*TestWorkflow, gcsPrefix, localPath string) error {
	log.Printf("Storing artifacts and logs in %s", gcsPrefix)
	logGCSDirectoryURL(gcsPrefix)

	for _, twf := range tests {
		if twf.wf == nil {
			return fmt.Errorf("found nil workflow in finalize")
		}

		twf.wf.StorageClient = client

		// $GCS_PATH/2021-04-20T11:44:08-07:00/image_validation/debian-10
		twf.GCSPath = fmt.Sprintf("%s/%s/%s", gcsPrefix, twf.Name, twf.Image.Name)
		twf.wf.GCSPath = twf.GCSPath

		// Process quota steps and associated creation steps.
		for quotaStepName, createStepName := range map[string]string{
			waitForVMQuotaStepName:    createVMsStepName,
			waitForDisksQuotaStepName: createDisksStepName,
		} {
			quotaStep, ok := twf.wf.Steps[quotaStepName]
			if !ok {
				continue
			}
			for _, q := range quotaStep.WaitForAvailableQuotas.Quotas {
				// Populate empty regions with best guess from the zone
				if q.Region == "" {
					lastIndex := strings.LastIndex(twf.wf.Zone, "-")
					q.Region = twf.wf.Zone[:lastIndex]
				}
			}
			createStep, ok := twf.wf.Steps[createStepName]
			if !ok {
				continue
			}
			// Fix dependencies. Create steps should depend on the quota step, and quota steps should inherit all other dependencies.
			for _, dep := range twf.wf.Dependencies[createStepName] {
				dStep, ok := twf.wf.Steps[dep]
				if ok {
					if err := twf.wf.AddDependency(quotaStep, dStep); err != nil {
						return err
					}
				}
			}
			if err := twf.wf.AddDependency(createStep, quotaStep); err != nil {
				return err
			}
		}

		for _, step := range twf.wf.Steps {
			if step.CreateInstances != nil {
				twf.configureNetwork(step)
				twf.configureExternalIP(step)
			}
		}

		// Assume amd64 when arch is not set.
		arch := "amd64"
		if twf.Image.Architecture == "ARM64" {
			arch = "arm64"
		}

		createDisksStep, createDisksOk := twf.wf.Steps[createDisksStepName]
		createVMsStep, ok := twf.wf.Steps[createVMsStepName]
		if ok {
			for _, vm := range createVMsStep.CreateInstances.Instances {
				if vm.MachineType != "" {
					log.Printf("VM %s machine type set to %s for test %s\n", vm.Name, vm.MachineType, twf.Name)
					if !twf.matchesWorkflowMachineType(vm.MachineType) && twf.ReservationAffinity != nil && twf.ReservationAffinity.ConsumeReservationType == "SPECIFIC_RESERVATION" {
						vm.ReservationAffinity = nil
						if vm.Scheduling != nil && vm.Scheduling.ProvisioningModel == "RESERVATION_BOUND" {
							vm.Scheduling.ProvisioningModel = ""
						}
					}
				} else {
					vm.MachineType = twf.MachineType.Name
				}
				if vm.Zone != "" && vm.Zone != twf.wf.Zone {
					// argZoneOverride means that the flag provided zone(s) will be used,
					// and the vm zone (hardcoded in the test suite) will be ignored.
					if twf.argZoneOverride {
						log.Printf("-zone_override is set to true, VM %s zone is set to %s, differing from workflow zone %s for test %s, not overriding\n", vm.Name, vm.Zone, twf.wf.Zone, twf.Name)
					} else {
						// argZoneOverride is false, so the vm zone (hardcoded in the test
						// suite) will be used.
						twf.wf.Zone = vm.Zone
						log.Printf("-zone_override is set to false, VM %s zone is set to %s, differing from workflow zone %s for test %s, overriding\n", vm.Name, vm.Zone, twf.wf.Zone, twf.Name)
					}
				}
				if createDisksOk {
					for _, attachedDisk := range vm.Disks {
						for _, disk := range *createDisksStep.CreateDisks {
							if attachedDisk.Source == disk.Name && disk.Type == "" {
								disk.Type = DiskTypeNeeded(twf.MachineType.Name)
							}
						}
					}
				}

				// Correct initialize params disk types for custom attached disks.
				for _, attachedDisk := range vm.Disks {
					if attachedDisk.InitializeParams != nil {
						// Default to standard if no disk type is specified.
						if attachedDisk.InitializeParams.DiskType != "" && !strings.HasPrefix(attachedDisk.InitializeParams.DiskType, "zones/") {
							// Qualify the disk type with the zone if it's missing.
							attachedDisk.InitializeParams.DiskType = fmt.Sprintf("zones/%s/diskTypes/%s", vm.Zone, attachedDisk.InitializeParams.DiskType)
						}
					}
				}
			}
			for _, vm := range createVMsStep.CreateInstances.InstancesBeta {
				if vm.MachineType != "" {
					log.Printf("VM %s machine type set to %s for test %s\n", vm.Name, vm.MachineType, twf.Name)
					if !twf.matchesWorkflowMachineType(vm.MachineType) && ((twf.ReservationAffinityBeta != nil && twf.ReservationAffinityBeta.ConsumeReservationType == "SPECIFIC_RESERVATION") || (twf.ReservationAffinity != nil && twf.ReservationAffinity.ConsumeReservationType == "SPECIFIC_RESERVATION")) {
						vm.ReservationAffinity = nil
						if vm.Scheduling != nil && vm.Scheduling.ProvisioningModel == "RESERVATION_BOUND" {
							vm.Scheduling.ProvisioningModel = ""
						}
					}
				} else {
					vm.MachineType = twf.MachineType.Name
				}
			}
		}

		if utils.HasFeature(twf.Image, "WINDOWS") {
			archBits := "64"
			if strings.Contains(twf.ImageURL, "x86") {
				archBits = "32"
			}
			twf.wf.Sources["testpackage"] = fmt.Sprintf("%s/%s%s.exe", localPath, twf.Name, archBits)
			twf.wf.Sources["wrapper.exe"] = fmt.Sprintf("%s/%s%s.exe", localPath, testWrapperPathWindows, archBits)
		} else {
			twf.wf.Sources["testpackage"] = fmt.Sprintf("%s/%s.%s.test", localPath, twf.Name, arch)
			twf.wf.Sources["wrapper"] = fmt.Sprintf("%s%s.%s", localPath, testWrapperPath, arch)
		}

		// add a final copy-objects step which copies the daisy-outs-path directory to twf.gcsPath + /outs
		copyGCSObject := daisy.CopyGCSObject{}
		copyGCSObject.Source = "${OUTSPATH}/" // Trailing slash apparently crucial.
		copyGCSObject.Destination = twf.GCSPath + "/outs"
		copyGCSObjects := &daisy.CopyGCSObjects{copyGCSObject}
		copyStep, err := twf.wf.NewStep("copy-objects")
		if err != nil {
			return fmt.Errorf("failed to add copy-objects step to workflow %s: %v", twf.Name, err)
		}
		copyStep.CopyGCSObjects = copyGCSObjects

		// The "copy-objects" step depends on every wait step.
		for stepname, step := range twf.wf.Steps {
			if !strings.HasPrefix(stepname, "wait-") {
				continue
			}
			if err := twf.wf.AddDependency(copyStep, step); err != nil {
				return fmt.Errorf("failed to add copy-objects step: %v", err)
			}
		}

	}
	return nil
}

type testResult struct {
	testWorkflow    *TestWorkflow
	skipped         bool
	workflowSuccess bool
	err             error
	results         []string
}

func getTestResults(ctx context.Context, ts *TestWorkflow) ([]string, error) {
	var results []string
	createVMsStep, ok := ts.wf.Steps[createVMsStepName]
	if ok {
		for _, vm := range createVMsStep.CreateInstances.Instances {
			out, err := utils.DownloadGCSObject(ctx, client, vm.Metadata["_test_results_url"])
			if err != nil {
				return nil, fmt.Errorf("failed to get results for test %s vm %s: %v", ts.Name, vm.Name, err)
			}
			results = append(results, string(out))
		}
		for _, vm := range createVMsStep.CreateInstances.InstancesBeta {
			out, err := utils.DownloadGCSObject(ctx, client, vm.Metadata["_test_results_url"])
			if err != nil {
				return nil, fmt.Errorf("failed to get results for test %s vm %s: %v", ts.Name, vm.Name, err)
			}
			results = append(results, string(out))
		}
	}

	return results, nil
}

// NewTestWorkflow returns a new TestWorkflow.
func NewTestWorkflow(opts *TestWorkflowOpts, setupFunc func(*TestWorkflow) error) (*TestWorkflow, error) {
	if len(opts.Zones) == 0 && opts.Zone != "" {
		opts.Zones = []string{opts.Zone}
	}
	t := &TestWorkflow{}
	t.opts = opts
	t.counter = 0
	t.Name = opts.Name
	t.ImageURL = opts.Image
	t.Client = opts.Client
	t.testExcludeFilter = opts.ExcludeFilter
	t.argZoneOverride = opts.ArgZoneOverride
	t.SetupFunc = setupFunc

	t.ReservationBound = opts.ReservationBound
	if opts.UseReservations {
		reservationType := "ANY_RESERVATION"
		var reservationKey string
		if len(opts.ReservationURLs) > 0 {
			reservationKey = "compute.googleapis.com/reservation-name"
			reservationType = "SPECIFIC_RESERVATION"
		}
		t.ReservationAffinity = &compute.ReservationAffinity{ConsumeReservationType: reservationType, Values: opts.ReservationURLs, Key: reservationKey}
		t.ReservationAffinityBeta = &computeBeta.ReservationAffinity{ConsumeReservationType: reservationType, Values: opts.ReservationURLs, Key: reservationKey}
	}
	t.AcceleratorType = opts.AcceleratorType
	t.AcceleratorCount = opts.AcceleratorCount

	var err error
	t.Project, err = t.Client.GetProject(opts.Project)
	if err != nil {
		return nil, err
	}
	t.Zone, err = t.Client.GetZone(t.Project.Name, opts.Zone)
	if err != nil {
		return nil, err
	}

	// Initializing Image inside the TestWorkflow
	split := strings.Split(opts.Image, "/")
	if strings.Contains(opts.Image, "family") {
		t.Image, err = t.Client.GetImageFromFamily(split[1], split[len(split)-1])
	} else {
		t.Image, err = t.Client.GetImage(split[1], split[len(split)-1])
	}

	// Initializing ImageBeta inside the TestWorkflow, this is required to provide Beta support to cvm testsuite
	if strings.Contains(opts.Image, "family") {
		t.ImageBeta, err = t.Client.GetImageFromFamilyBeta(split[1], split[len(split)-1])
	} else {
		t.ImageBeta, err = t.Client.GetImageBeta(split[1], split[len(split)-1])
	}
	if err != nil {
		return nil, err
	}
	if t.Image.Architecture == "ARM64" {