cq/scripts/slave/step/fuchsia_emulator_step.py - manifest_repos/continuous-tests - Git at Google

 """Step class for managing a Fuchsia emulator instance using ffx emu."""

 import logging
 import os
 import shutil
 import socket

 from unittest import mock

 from slave import base_step


 _STEP_DATA_FEMU_CAST = 'step_data_femu_cast'
 _STEP_DATA_FEMU_HTTP = 'step_data_femu_http'
 _STEP_DATA_FEMU_HTTPS = 'step_data_femu_https'
 _STEP_DATA_FEMU_SSH = 'step_data_femu_ssh'
 _STEP_DATA_FEMU_LOG = 'step_data_femu_log'
 _STEP_DATA_PACKAGE_SERVER_LOG = 'step_data_package_server_log'

 _FFX_LOG = 'ffx.log'
 _FFX_DAEMON_LOG = 'ffx.daemon.log'

 _SSH_CONFIG_TEMPLATE = '''
 # Generated by Eureka CQ FuchsiaEmulatorStep using ffx emu.
 # Ports are changed to an available TCP port assigned to QEMU.
 Host *
   Port {port}
   StrictHostKeyChecking no
   IdentityFile {auth_path}
   ControlPersist 1m
   ControlMaster auto
   ControlPath /tmp/ssh-%r@%h:%p
   ServerAliveInterval 1
   ServerAliveCountMax 3
   ConnectTimeout 5
 '''

 class FuchsiaEmulatorError(RuntimeError):
   """Errors relating to FEMU."""

 class BaseFuchsiaEmulatorStep(base_step.BaseStep):
   """BaseFuchsiaEmulatorStep with common utilities."""
   def __init__(self,
                fuchsia_sdk_root,
                fuchsia_workdir,
                product='terminal.qemu-x64',
                name='Unknown FEMU step',
                **kwargs):
     """Creates a BaseFuchsiaEmulatorStep instance.

     Args:
       fuchsia_sdk_root: Path to the Fuchsia SDK.
       fuchsia_workdir: Path to directory with Fuchsia artifacts.
       product: Product image to run the emulator on.
       name: Name of the step to pass to BaseStep.
       kwargs: Remaining args to pass to BaseStep.
     """
     base_step.BaseStep.__init__(self, name=name, **kwargs)
     self._fuchsia_sdk_root = fuchsia_sdk_root
     prod, board = product.split('.', 1)
     self._product_bundle = os.path.join(fuchsia_sdk_root, 'images', prod, board)
     # GN SDK scripts
     self._ffx = os.path.join(self._fuchsia_sdk_root, 'sdk/tools/x64/ffx')

     # Common Paths
     self._fuchsia_workdir_path = fuchsia_workdir

   def _dump_file_to_log(self, file_path):
     """Dump file to log.

     Args:
         file_path: Location of the file to add to the logger output.
     """
     logging.info('%s log:', file_path)
     with open(file_path) as file_fd:
       logging.info(file_fd.read())


 class FuchsiaEmulatorStartStep(BaseFuchsiaEmulatorStep):
   """Step for starting local Fuchsia assets on FEMU."""

   def __init__(self, **kwargs):
     """Creates a FuchsiaEmulatorCreateStep instance.

     Args:
       kwargs: Passed to FuchsiaEmulatorBaseStep.
     """
     BaseFuchsiaEmulatorStep.__init__(self, name="Start FEMU", **kwargs)
     self.femu_log = None
     self.package_server_log = None
     self._ports_in_use = []

   def _get_available_tcp_port(self, max_attempts=100):
     """Find a unique and open port.

     Verify port is available by opening a listening socket on it.

     Verify uniqueness by storing a list of ports returned, and ensuring no
     duplicates in this list.
     """
     # Search for a port that is open and verify it isn't
     # already being used for another FEMU port.
     attempts = 0
     while attempts < max_attempts:
       attempts += 1
       # Look for an open port
       sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
       sock.bind(("", 0))
       port = sock.getsockname()[1]
       sock.close()
       # Verify port is not being used for another port
       if port not in self._ports_in_use:
         break
     if attempts >= max_attempts:
       raise FuchsiaEmulatorError('Failed to find open tcp port')
     self._ports_in_use.append(port)
     return port

   def write_file(self, file, content):
     with open(file, 'w') as f:
       f.write(content)

   def create_ssh_config(self):
     """Create SSH config for the test runner to connect to FEMU.

     Assumes that FEMU has already been started.
     """
     home_dir = os.path.expanduser('~')
     ssh_dir = os.path.join(home_dir, '.ssh')
     returncode, _, _ = self.exec_subprocess(['mkdir', '-p', ssh_dir])
     if returncode != 0:
       self._dump_file_to_log(self.femu_log)
       self._dump_file_to_log(self.package_server_log)
       raise FuchsiaEmulatorError('Failed to create ssh out dir')
     ssh_config_path = os.path.join(home_dir, '.fuchsia', 'sshconfig')
     ssh_config_content = _SSH_CONFIG_TEMPLATE.format(
       port=self._ssh_port,
       auth_path=os.path.join(ssh_dir, 'fuchsia_ed25519'))
     self.write_file(ssh_config_path, ssh_config_content)
     logging.info('ssh_config for chromium test at %s', ssh_config_path)

   def start_femu(self):
     """Start Fuchsia Emulator process."""
     self._ssh_port = self._get_available_tcp_port()

     _, self.femu_log = self.get_new_tmp_file()
     logging.info('femu log at %s', self.femu_log)
     self.add_step_data(_STEP_DATA_FEMU_LOG, self.femu_log)

     _, self.package_server_log = self.get_new_tmp_file()
     logging.info('package server log at %s', self.package_server_log)
     self.add_step_data(_STEP_DATA_PACKAGE_SERVER_LOG, self.package_server_log)

     # HACK(b/290049007)
     # For currently unknown reasons, there is an ffx daemon socket already
     # present, but occassionally unresponsive. The emu start command will
     # fail if it attempts to use this bad daemon. As a stop-gap measure
     # to get Eureka CQ/CB back to green, we will force restart the daemon.
     ffx_daemon_stop = [
         self._ffx,
         '-c', 'log.level=debug',
         '-c', 'log.dir={}'.format(self._fuchsia_workdir_path),
         'daemon',
         'stop',
         '-t', '5000',
     ]
     self.exec_subprocess(ffx_daemon_stop, check_output=True)

     # Start FEMU
     femu_command = [
         self._ffx,
         '--timeout', '100',
         '-c', 'log.level=debug',
         '-c', 'log.dir={}'.format(self._fuchsia_workdir_path),
         'emu',
         'start',
         '--startup-timeout', '100',
         self._product_bundle,
         '--log', self.femu_log,
         '--port-map', f'ssh:{self._ssh_port}',
         '--headless',  # Run without gui on infra
     ]
     self.exec_subprocess(femu_command, check_output=True)

   def run(self):
     """Starts a local FEMU instance."""
     self.start_femu()
     # Copy SSH config needed for chromium test runners.
     self.create_ssh_config()
     # Output logs to give additional debugging information.
     self._dump_file_to_log(self.femu_log)
     # Store femu ports in step data for test steps.
     self.add_step_data(_STEP_DATA_FEMU_SSH, self._ssh_port)
     return True


 class FuchsiaEmulatorTeardownStep(BaseFuchsiaEmulatorStep):
   """Step for cleaning up after the FuchsiaEmulatorStartStep."""

   def __init__(self, **kwargs):
     """Creates a FuchsiaEmulatorTeardownStep instance."""
     BaseFuchsiaEmulatorStep.__init__(self, name="Teardown FEMU", **kwargs)

   def _copy_logs_to_gcs(self):
     """Copies FuchsiaEmulator related logs to the GCS directory"""
     log_source_files_to_destinations = {
         self.get_step_data(_STEP_DATA_FEMU_LOG): 'femu_system_log.txt',
         self.get_step_data(
             _STEP_DATA_PACKAGE_SERVER_LOG
         ): 'package_server_log.txt',
         os.path.join(self._fuchsia_workdir_path, _FFX_LOG): 'ffx.log',
         os.path.join(
             self._fuchsia_workdir_path, _FFX_DAEMON_LOG
         ): 'ffx.daemon.log',
     }

     for src, dst in log_source_files_to_destinations.items():
       if not os.path.isfile(src):
         logging.warning('The expected log file "%s" does not exist!', src)
         continue

       gcs_upload_dst = os.path.join(self.get_gcs_dir(), dst)
       shutil.copyfile(src, gcs_upload_dst)

   def run(self):
     """Cleanup after FuchsiaEmulatorTeardownStep."""
     # Kill FEMU
     self._copy_logs_to_gcs()
     femu_command = [self._ffx,
                     'emu',
                     'stop',
                     '--all',
                     ]
     # Don't check the return code. If emulator crashed, ffx emu will
     # throw an error but it doesn't mean the kill action failed.
     self.exec_subprocess(femu_command)
     return True
	"""Step class for managing a Fuchsia emulator instance using ffx emu."""

	import logging
	import os
	import shutil
	import socket

	from unittest import mock

	from slave import base_step


	_STEP_DATA_FEMU_CAST = 'step_data_femu_cast'
	_STEP_DATA_FEMU_HTTP = 'step_data_femu_http'
	_STEP_DATA_FEMU_HTTPS = 'step_data_femu_https'
	_STEP_DATA_FEMU_SSH = 'step_data_femu_ssh'
	_STEP_DATA_FEMU_LOG = 'step_data_femu_log'
	_STEP_DATA_PACKAGE_SERVER_LOG = 'step_data_package_server_log'

	_FFX_LOG = 'ffx.log'
	_FFX_DAEMON_LOG = 'ffx.daemon.log'

	_SSH_CONFIG_TEMPLATE = '''
	# Generated by Eureka CQ FuchsiaEmulatorStep using ffx emu.
	# Ports are changed to an available TCP port assigned to QEMU.
	Host *
	Port {port}
	StrictHostKeyChecking no
	IdentityFile {auth_path}
	ControlPersist 1m
	ControlMaster auto
	ControlPath /tmp/ssh-%r@%h:%p
	ServerAliveInterval 1
	ServerAliveCountMax 3
	ConnectTimeout 5
	'''

	class FuchsiaEmulatorError(RuntimeError):
	"""Errors relating to FEMU."""

	class BaseFuchsiaEmulatorStep(base_step.BaseStep):
	"""BaseFuchsiaEmulatorStep with common utilities."""
	def __init__(self,
	fuchsia_sdk_root,
	fuchsia_workdir,
	product='terminal.qemu-x64',
	name='Unknown FEMU step',
	**kwargs):
	"""Creates a BaseFuchsiaEmulatorStep instance.

	Args:
	fuchsia_sdk_root: Path to the Fuchsia SDK.
	fuchsia_workdir: Path to directory with Fuchsia artifacts.
	product: Product image to run the emulator on.
	name: Name of the step to pass to BaseStep.
	kwargs: Remaining args to pass to BaseStep.
	"""
	base_step.BaseStep.__init__(self, name=name, **kwargs)
	self._fuchsia_sdk_root = fuchsia_sdk_root
	prod, board = product.split('.', 1)
	self._product_bundle = os.path.join(fuchsia_sdk_root, 'images', prod, board)
	# GN SDK scripts
	self._ffx = os.path.join(self._fuchsia_sdk_root, 'sdk/tools/x64/ffx')

	# Common Paths
	self._fuchsia_workdir_path = fuchsia_workdir

	def _dump_file_to_log(self, file_path):
	"""Dump file to log.

	Args:
	file_path: Location of the file to add to the logger output.
	"""
	logging.info('%s log:', file_path)
	with open(file_path) as file_fd:
	logging.info(file_fd.read())


	class FuchsiaEmulatorStartStep(BaseFuchsiaEmulatorStep):
	"""Step for starting local Fuchsia assets on FEMU."""

	def __init__(self, **kwargs):
	"""Creates a FuchsiaEmulatorCreateStep instance.

	Args:
	kwargs: Passed to FuchsiaEmulatorBaseStep.
	"""
	BaseFuchsiaEmulatorStep.__init__(self, name="Start FEMU", **kwargs)
	self.femu_log = None
	self.package_server_log = None
	self._ports_in_use = []

	def _get_available_tcp_port(self, max_attempts=100):
	"""Find a unique and open port.

	Verify port is available by opening a listening socket on it.

	Verify uniqueness by storing a list of ports returned, and ensuring no
	duplicates in this list.
	"""
	# Search for a port that is open and verify it isn't
	# already being used for another FEMU port.
	attempts = 0
	while attempts < max_attempts:
	attempts += 1
	# Look for an open port
	sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	sock.bind(("", 0))
	port = sock.getsockname()[1]
	sock.close()
	# Verify port is not being used for another port
	if port not in self._ports_in_use:
	break
	if attempts >= max_attempts:
	raise FuchsiaEmulatorError('Failed to find open tcp port')
	self._ports_in_use.append(port)
	return port

	def write_file(self, file, content):
	with open(file, 'w') as f:
	f.write(content)

	def create_ssh_config(self):
	"""Create SSH config for the test runner to connect to FEMU.

	Assumes that FEMU has already been started.
	"""
	home_dir = os.path.expanduser('~')
	ssh_dir = os.path.join(home_dir, '.ssh')
	returncode, _, _ = self.exec_subprocess(['mkdir', '-p', ssh_dir])
	if returncode != 0:
	self._dump_file_to_log(self.femu_log)
	self._dump_file_to_log(self.package_server_log)
	raise FuchsiaEmulatorError('Failed to create ssh out dir')
	ssh_config_path = os.path.join(home_dir, '.fuchsia', 'sshconfig')
	ssh_config_content = _SSH_CONFIG_TEMPLATE.format(
	port=self._ssh_port,
	auth_path=os.path.join(ssh_dir, 'fuchsia_ed25519'))
	self.write_file(ssh_config_path, ssh_config_content)
	logging.info('ssh_config for chromium test at %s', ssh_config_path)

	def start_femu(self):
	"""Start Fuchsia Emulator process."""
	self._ssh_port = self._get_available_tcp_port()

	_, self.femu_log = self.get_new_tmp_file()
	logging.info('femu log at %s', self.femu_log)
	self.add_step_data(_STEP_DATA_FEMU_LOG, self.femu_log)

	_, self.package_server_log = self.get_new_tmp_file()
	logging.info('package server log at %s', self.package_server_log)
	self.add_step_data(_STEP_DATA_PACKAGE_SERVER_LOG, self.package_server_log)

	# HACK(b/290049007)
	# For currently unknown reasons, there is an ffx daemon socket already
	# present, but occassionally unresponsive. The emu start command will
	# fail if it attempts to use this bad daemon. As a stop-gap measure
	# to get Eureka CQ/CB back to green, we will force restart the daemon.
	ffx_daemon_stop = [
	self._ffx,
	'-c', 'log.level=debug',
	'-c', 'log.dir={}'.format(self._fuchsia_workdir_path),
	'daemon',
	'stop',
	'-t', '5000',
	]
	self.exec_subprocess(ffx_daemon_stop, check_output=True)

	# Start FEMU
	femu_command = [
	self._ffx,
	'--timeout', '100',
	'-c', 'log.level=debug',
	'-c', 'log.dir={}'.format(self._fuchsia_workdir_path),
	'emu',
	'start',
	'--startup-timeout', '100',
	self._product_bundle,
	'--log', self.femu_log,
	'--port-map', f'ssh:{self._ssh_port}',
	'--headless', # Run without gui on infra
	]
	self.exec_subprocess(femu_command, check_output=True)

	def run(self):
	"""Starts a local FEMU instance."""
	self.start_femu()
	# Copy SSH config needed for chromium test runners.
	self.create_ssh_config()
	# Output logs to give additional debugging information.
	self._dump_file_to_log(self.femu_log)
	# Store femu ports in step data for test steps.
	self.add_step_data(_STEP_DATA_FEMU_SSH, self._ssh_port)
	return True


	class FuchsiaEmulatorTeardownStep(BaseFuchsiaEmulatorStep):
	"""Step for cleaning up after the FuchsiaEmulatorStartStep."""

	def __init__(self, **kwargs):
	"""Creates a FuchsiaEmulatorTeardownStep instance."""
	BaseFuchsiaEmulatorStep.__init__(self, name="Teardown FEMU", **kwargs)

	def _copy_logs_to_gcs(self):
	"""Copies FuchsiaEmulator related logs to the GCS directory"""
	log_source_files_to_destinations = {
	self.get_step_data(_STEP_DATA_FEMU_LOG): 'femu_system_log.txt',
	self.get_step_data(
	_STEP_DATA_PACKAGE_SERVER_LOG
	): 'package_server_log.txt',
	os.path.join(self._fuchsia_workdir_path, _FFX_LOG): 'ffx.log',
	os.path.join(
	self._fuchsia_workdir_path, _FFX_DAEMON_LOG
	): 'ffx.daemon.log',
	}

	for src, dst in log_source_files_to_destinations.items():
	if not os.path.isfile(src):
	logging.warning('The expected log file "%s" does not exist!', src)
	continue

	gcs_upload_dst = os.path.join(self.get_gcs_dir(), dst)
	shutil.copyfile(src, gcs_upload_dst)

	def run(self):
	"""Cleanup after FuchsiaEmulatorTeardownStep."""
	# Kill FEMU
	self._copy_logs_to_gcs()
	femu_command = [self._ffx,
	'emu',
	'stop',
	'--all',
	]
	# Don't check the return code. If emulator crashed, ffx emu will
	# throw an error but it doesn't mean the kill action failed.
	self.exec_subprocess(femu_command)
	return True