catatester/puppet/cast_builder/manifests/swarm_host_base.pp

# Common setup for all Swarming device-bot hosts for CI

class cast_builder::swarm_host_base ($user, $group, $home_dir) {

  # save globals as in-scope class variables
  $cipd_ref           = $cipd_env  # lint:ignore:variable_scope

  # define class variables
  $cipd_key           = "${home_dir}/home-cipd-service-account.json"
  $gsutil_dir         = "${home_dir}/gsutil"
  $gsutil_boto        = "${home_dir}/.boto"
  $fragglerock_key    = "${home_dir}/fragglerock_service_key.json"
  $swarm_home_dir     = "${home_dir}/swarm_home"
  $depot_tools_dir    = "${home_dir}/depot_tools"
  $depot_tools_remote = 'https://chromium.googlesource.com/chromium/tools/depot_tools.git'
  $swarm_bots_home    = "${swarm_home_dir}/bots"
  $venv               = "${swarm_home_dir}/swarm_env"
  $email              = 'livingroom-releng+alerts+puppet@google.com'

  $common_packages = [
      'alsa-utils',  # Provides "arecord" and "amixer", used by MZ sync tests.
      'avahi-utils',  # Provides "avahi-browse", for MZ discovery tests.
      'build-essential',
      'curl',
      'git',
      'libcurl4-gnutls-dev',
      'libffi-dev',
      'libfreetype6-dev',
      'libsox-fmt-mp3',
      'lm-sensors',
      'nodejs',
      'python-dev',
      'python-oauth2client',
      'python-setuptools',
      'python-virtualenv',
      'sox',
      'tar',
      'vim',
  ]

  # For debian 10.0 libav-tools is replaced by ffmpeg
  $major_os_release = 0.0 + $::operatingsystemmajrelease
  if $major_os_release >= 10.0 {
    $packages = $common_packages + ['ffmpeg']
  }
  else {
    $packages = $common_packages + ['libav-tools']
  }

  package { $packages:
    ensure  => present,
    require => Exec['apt-get_update'],
  }

  Exec {
    # git needs HOME to know where to write the global config.
    environment => ["HOME=${home_dir}"],
    path        => ['/usr/local/sbin', '/usr/local/bin', '/usr/sbin', '/usr/bin', '/sbin', '/bin', $depot_tools_dir],
    user        => $user,
  }

  File {
    owner => $user,
    group => $group,
  }

  exec { 'apt-get_update':
    user    => 'root',
    group   => 'root',
    command => '/usr/bin/apt-get update',
    timeout => 300,
  }

  group { $group:
      ensure => 'present',
  }

  file { $home_dir:
    ensure => directory,
    mode   => '0755'
  }

  file { $swarm_bots_home:
    ensure => directory,
    mode   => '0755'
  }

  file { $cipd_key:
    ensure => file,
    mode   => '0750'
  } ->
  cast_builder::test_md5 { $cipd_key:
    md5 => $::md5_cipd_key,
  } ->
  notify { 'has_cipd_key':
    message => "cipd file exists at ${cipd_key}",
  }

  exec { 'clone_depot_tools_repo':
    cwd     => $home_dir,
    command => "git clone ${depot_tools_remote}",
    creates => $depot_tools_dir,
    require => Package['git'],
  } ->
  exec { 'git_pull_depot_tools':
    cwd     => $depot_tools_dir,
    command => 'git pull origin master',
  }

  ### Pip & virtualenv setup

  file { $venv:
    ensure  => directory,
    mode    => '0755',
    require => File[$home_dir],
  }

  $pip_cmd = "${venv}/bin/pip"
  exec { 'create_python_virtualenv':
    command => "virtualenv ${venv}; ${pip_cmd} --log ${venv}/pip.log install --upgrade pip setuptools",
    creates => "${venv}/bin/activate",
    cwd     => $home_dir,
    # Unless activate exists and VIRTUAL_ENV is correct we re-create the virtualenv
    unless  => "grep '^[\\t ]*VIRTUAL_ENV=[\\\\'\\\"]*${venv}[\\\"\\\\'][\\t ]*$' ${venv}/bin/activate",
    require => [
        Package['python-virtualenv'],
        File[$venv]
    ],
  } ->
  # Enable $user to write to its own .cache directory.
  file { "${home_dir}/.cache":
    ensure => 'directory',
    owner  => $user,
    group  => $group,
  } ->
  exec { 'pip_install_requirements':
    command => "bash -c 'source ${venv}/bin/activate && ${pip_cmd} install -U pip wheel &&
                ${pip_cmd} install pyyaml'",
    require => [
        Exec['create_python_virtualenv'],
        Package['python-virtualenv'],
    ],
    timeout => 120,
  }

  ### Ensure that gsutil is set up. Some tests use it to copy files to a media server.
  # File with no source will be created with 0 size by puppet if missing
  # Manual deployment of .boto and .p12 file required for new host
  # (see instructions in swarm_init.pp)
  file { $gsutil_boto:
    ensure  => file,
    mode    => '0755',
    require => File[$home_dir],
  } ->
  cast_builder::test_md5 { $gsutil_boto:
    md5 => $::md5_gsutil_boto,
  } ->
  notify { 'correct_boto':
    message => "Successfully tested file ${home_dir}/.boto",
  }

  # File with no source will be created with 0 size by puppet if missing
  # Manual deployment of json key file required for new host
  # (see instructions in swarm_init.pp)
  file { $fragglerock_key:
    ensure  => file,
    mode    => '0755',
    require => File[$home_dir],
  } ->
  cast_builder::test_md5 { $fragglerock_key:
    md5 => $::md5_fragglerock_key,
  } ->
  notify { 'correct_fragglerock_key_file':
    message => "Successfully tested file ${fragglerock_key}",
  }

  # Get, install, update, and configure gsutil
  file { $gsutil_dir:
    ensure  => directory,
    mode    => '0755',
    require => Notify['correct_boto'],
  } ->
  exec { 'get gsutil':
    cwd     => $home_dir,
    command => 'curl https://storage.googleapis.com/pub/gsutil.tar.gz -f -o gsutil.tar.gz',
    require => Package['curl'],
  } ->
  exec { 'install gsutil':
    cwd     => $home_dir,
    command => "tar --ungzip --extract --strip-components=2 --file=gsutil.tar.gz --directory=${gsutil_dir}",
    require => Package['tar'],
  } ->
  file { "${home_dir}/gsutil.tar.gz":
    ensure => absent,
  # Disable gsutil update until a safe release is available. b/135462553
  # } ->
  # # gsutil will not run but return 0 status with empty .boto file
  # exec { 'update gsutil':
  #   cwd     => $home_dir,
  #   command => "${gsutil_dir}/gsutil update -f -n",
  }
}