| """Build step class for uploading artifacts to GCS bucket.""" |
| from __future__ import absolute_import |
| import logging |
| import os |
| |
| from helpers import gcs_utils |
| from helpers import repo_utils |
| from slave import base_step |
| |
| |
| class GcsUploadStep(base_step.BaseStep): |
| """Build step class for uploading artifacts to GCS bucket.""" |
| |
| def __init__(self, gcs_bucket, gcs_path, src_path=None, skip_empty=False, |
| name='gcs_upload', result_property='gcs_output', **kwargs): |
| """Creates a GcsUploadStep instance. |
| |
| Step will upload from src_path to bigstore |
| The bigstore destination will be gs://<gcs_bucket>/<gcs_path> |
| If src_path pointing to dir, content of dir (e.g. file.txt) will be |
| uploaded to GCS as gs://<gcs_bucket>/<gcs_path>/file.txt |
| If src_path pointing to file (e.g. file.jpg) and gcs_path have trailing |
| slash, file will be uploaded gs://<gcs_bucket>/<gcs_path>/file.jpg |
| If src_path pointing to file (e.g. file.txt) and gcs_path does not have |
| trailing slash, than last component of gcs_path will become new name of |
| the uploaded object, file will be uploaded to gs://<gcs_bucket>/<gcs_path> |
| |
| Args: |
| gcs_bucket: destination bucket name. |
| gcs_path: relative path in bucket to upload src_path. |
| src_path: local path to file or dir upload, default to get_gcs_dir(). |
| skip_empty: if true step will not proceed and return success when |
| src_path does not exists or epmty dir. |
| name: step name. |
| result_property: name of property to save upload location and result. |
| **kwargs: Any additional args to pass to BaseStep. |
| """ |
| self._src_path = src_path |
| self._skip_empty = skip_empty |
| self._gcs_bucket = gcs_bucket |
| self._gcs_path = gcs_path |
| self._result_property = result_property |
| base_step.BaseStep.__init__(self, name=name, **kwargs) |
| |
| def run(self): |
| """Uploading to bigstore. |
| |
| Returns: |
| True if upload completed successfully. |
| """ |
| |
| if not self._src_path: |
| self._src_path = self.get_gcs_dir() |
| if self._skip_empty: |
| if not os.path.exists(self._src_path): |
| logging.info('Path %s does not exist, skipping', self._src_path) |
| return True |
| if os.path.isdir(self._src_path) and not os.listdir(self._src_path): |
| logging.info('Directory %s is empty, skipping', self._src_path) |
| return True |
| |
| if os.path.isdir(self._src_path): |
| path_to_upload = os.path.join(self._src_path, '.') |
| gcs_path = self._gcs_path |
| else: |
| path_to_upload = self._src_path |
| gcs_path = os.path.join(self._gcs_path, os.path.basename(self._src_path)) |
| |
| gcs_uri = 'gs://{gcs_bucket}/{gcs_path}'.format( |
| gcs_bucket=self._gcs_bucket, gcs_path=gcs_path) |
| command = gcs_utils.upload_directory_cmd( |
| path_to_upload, gcs_uri) |
| with repo_utils.normalize_manifest_if_exists( |
| self.get_property('manifest_override_path')): |
| returncode, _, _ = self.exec_subprocess(command) |
| gcs_output = { |
| 'uri': gcs_uri, |
| 'path': self._gcs_path, |
| 'bucket': self._gcs_bucket, |
| 'result': returncode, |
| } |
| self.set_build_property(self._result_property, gcs_output) |
| if returncode != 0: |
| return False |
| return True |
