hpc-slurm-h4d.yaml

# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
blueprint_name: slurm-h4d
vars:
  project_id: ## Set GCP Project ID Here ##
  deployment_name: slurm-h4d
  region: us-central1
  zone: us-central1-a
  rdma_net_range: 192.168.128.0/18
  instance_image:
    # Rocky 8: slurm-gcp-6-12-hpc-rocky-linux-8
    # Rocky 9: slurm-gcp-6-12-hpc-rocky-linux-9
    family: slurm-gcp-6-12-hpc-rocky-linux-9
    project: schedmd-slurm-public
  h4d_cluster_size: 2

  #Provisioning model, select one, lack of selection will rely on on-demand capacity
  h4d_reservation_name: ""
  h4d_dws_flex_enabled: false
  h4d_enable_spot_vm: false

# Documentation for each of the modules used below can be found at
# https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/main/modules/README.md
deployment_groups:
- group: primary
  modules:

  # Source is an embedded module, denoted by "modules/*" without ./, ../, /
  # as a prefix. To refer to a local module, prefix with ./, ../ or /

  - id: h4d-slurm-net-0
    source: modules/network/vpc
    settings:
      network_name: $(vars.deployment_name)-net

  - id: h4d-rdma-net
    source: modules/network/vpc
    settings:
      network_name: $(vars.deployment_name)-rdma-net-0
      mtu: 8896
      network_profile: https://www.googleapis.com/compute/beta/projects/$(vars.project_id)/global/networkProfiles/$(vars.zone)-vpc-falcon
      network_routing_mode: REGIONAL
      enable_cloud_router: false
      enable_cloud_nat: false
      enable_internal_traffic: false
      subnetworks:
      - subnet_name: $(vars.deployment_name)-rdma-sub-0
        subnet_region: $(vars.region)
        subnet_ip: $(vars.rdma_net_range)
        region: $(vars.region)
      nic_type: "IRDMA"

  - id: homefs
    source: modules/file-system/filestore
    use: [h4d-slurm-net-0]
    settings:
      filestore_tier: BASIC_SSD
      size_gb: 2560
      filestore_share_name: homeshare
      local_mount: /home

  - id: h4d_startup
    source: modules/scripts/startup-script
    settings:
      set_ofi_cloud_rdma_tunables: true
      local_ssd_filesystem:
        fs_type: ext4
        mountpoint: /mnt/lssd
        permissions: "1777"

  - id: h4d_nodeset
    source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset
    use: [h4d_startup, h4d-slurm-net-0]
    settings:
      # Unattended upgrades are disabled to prevent automatic daily updates which may lead to potential instability.
      # For more details, see https://cloud.google.com/compute/docs/instances/create-hpc-vm#disable_automatic_updates
      allow_automatic_updates: false
      bandwidth_tier: gvnic_enabled
      machine_type: h4d-highmem-192-lssd
      node_count_static: $(vars.h4d_cluster_size)
      node_count_dynamic_max: 0
      enable_placement: false
      enable_spot_vm: $(vars.h4d_enable_spot_vm)

      #Provisioning models
      reservation_name: $(vars.h4d_reservation_name)
      dws_flex:
        enabled: $(vars.h4d_dws_flex_enabled)

      disk_type: hyperdisk-balanced
      on_host_maintenance: TERMINATE
      additional_networks: $(h4d-rdma-net.instance_additional_networks)

  - id: h4d_partition
    source: community/modules/compute/schedmd-slurm-gcp-v6-partition
    use:
    - h4d_nodeset
    settings:
      exclusive: false
      partition_name: h4d
      is_default: true
      partition_conf:
        ResumeTimeout: 900
        SuspendTimeout: 600

  - id: slurm_login
    source: community/modules/scheduler/schedmd-slurm-gcp-v6-login
    use: [h4d-slurm-net-0]
    settings:
      machine_type: n2-standard-4
      enable_login_public_ips: true

  - id: controller_startup
    source: modules/scripts/startup-script
    settings:
      runners:
      - type: shell
        destination: stage_scripts.sh
        content: |
          #!/bin/bash
          SLURM_ROOT=/opt/apps/adm/slurm
          PARTITION_NAME=$(h4d_partition.partitions[0].partition_name)
          mkdir -m 0755 -p "${SLURM_ROOT}/scripts"

          # enable the use of password-free sudo within Slurm jobs on all compute nodes
          # feature is restricted to users with OS Admin Login IAM role
          # https://cloud.google.com/iam/docs/understanding-roles#compute.osAdminLogin
          mkdir -p "${SLURM_ROOT}/prolog_slurmd.d"
          mkdir -p "${SLURM_ROOT}/epilog_slurmd.d"
          curl -s -o "${SLURM_ROOT}/scripts/sudo-oslogin" \
              https://raw.githubusercontent.com/GoogleCloudPlatform/slurm-gcp/master/tools/prologs-epilogs/sudo-oslogin
          chmod 0755 "${SLURM_ROOT}/scripts/sudo-oslogin"

          ln -s "${SLURM_ROOT}/scripts/sudo-oslogin" "${SLURM_ROOT}/prolog_slurmd.d/sudo-oslogin.prolog_slurmd"
          ln -s "${SLURM_ROOT}/scripts/sudo-oslogin" "${SLURM_ROOT}/epilog_slurmd.d/sudo-oslogin.epilog_slurmd"

          # enable an RDMA health check that runs prior to any H4D job
          mkdir -p "${SLURM_ROOT}/partition-${PARTITION_NAME}-prolog_slurmd.d"
          curl -s -o "${SLURM_ROOT}/partition-${PARTITION_NAME}-prolog_slurmd.d/irdma_health_check.prolog_slurmd" \
              https://raw.githubusercontent.com/GoogleCloudPlatform/slurm-gcp/master/tools/prologs-epilogs/irdma_health_check
          chmod 0755 "${SLURM_ROOT}/partition-${PARTITION_NAME}-prolog_slurmd.d/irdma_health_check.prolog_slurmd"


  - id: slurm_controller
    source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller
    use: [h4d-slurm-net-0, h4d_partition, slurm_login, homefs]
    settings:
      enable_controller_public_ips: true
      controller_startup_script: $(controller_startup.startup_script)
      enable_external_prolog_epilog: true
      cloud_parameters:
        unkillable_step_timeout: 900