recipes/recipe_modules/archive/properties.proto

// Copyright 2020 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

syntax = "proto3";
package recipe_modules.build.archive;

// Next id: 4
message LatestFile {
  // The file for the latest file. Format is the same as |ArchiveData.gcs_path|.
  string gcs_path = 1;
  // The content of the latest file. It supports the same string
  // substitutions as |ArchiveData.gcs_path|.
  string gcs_file_content = 2;
  // The bucket to upload latest file. Format is the same as
  // |ArchiveData.gcs_bucket|.
  // If set, latest file will be uploaded to a separate bucket that other
  // archives.
  // This supports the same string substitutions as |ArchiveData.gcs_path|.
  string gcs_bucket = 3;
}

// Next id: 2
message RevisionsFile {
  // The file for the revisions file. Format is the same as |ArchiveData.gcs_path|.
  string gcs_path = 1;
}

// Next id: 3
message ArchiveFileRename {
  string from_file = 1;
  // to_file supports the same string substitutions as |ArchiveData.gcs_path|.
  string to_file = 2;
}

// Next id: 3
message ArchiveDirRename {
  string from_dir = 1;
  // to_dir supports the same string substitutions as |ArchiveData.gcs_path|.
  string to_dir = 2;
}

// Next id: 3
message SquashfsParams {
  // If not set, it will use default compression algorithm gzip.
  // Currently this can only be set to 'zstd'.
  string algorithm = 1;
  // Set the block_size. If not set, it will use default value 128K.
  // Example value: 256K, 1M.
  // Source of truth:
  // https://github.com/plougher/squashfs-tools/blob/4.5.1/README-4.5.1#L250
  string block_size = 2;
  // If not set, it uses default level for gzip, or 22(highest) for zstd.
  int32 compression_level = 3;
}

// Next id: 2
message TarZstdParams {
  // If not set, it uses default level.
  int32 compression_level = 1;
}


// Next id: 23
message ArchiveData {
  // A list of relative paths to files that should be archived. All files are
  // relative to the build directory, which is passed alongside this proto to
  // the archiving code.
  repeated string files = 1;
  // Rename files. Internally renaming happens after copying all files to a
  // temporary folder and before archiving. e.g. If the compiled target
  // has a file named "foo.zoo", and you want it to be "foo" in the zip file,
  // then you can have:
  //   files = ["foo.zoo"]
  //   rename_files = [{ "from_file":"foo.zoo", "to_file":"foo" }]
  repeated ArchiveFileRename rename_files = 10;
  // A list of globs that will be expanded to files that should be archived. All
  // globs are expanded relative to the build directory, which is passed
  // alongside this proto to the archiving code.
  repeated string file_globs = 6;
  // A list of relative paths to directories that should be archived. All
  // directories are relative to the build directory, which is passed alongside
  // this proto to the archiving code.
  // Note: The |files| and |dirs| fields don't interact. Both are relative to
  // the build directory.
  repeated string dirs = 2;
  // Rename dirs. Internally renaming happens after copying all files to a
  // temporary folder and before archiving. e.g. If the build output
  // has a path named "foo", and you want it to be "bar" in the zip file,
  // then you can have:
  //   dirs= ["foo"]
  //   rename_dirs = [{ "from_dir":"foo", "to_dir":"bar" }]
  // NOTE: If 'to_dir' already exists, 'from_dir' will become a subdir of
  // 'to_dir', instead of replacing 'to_dir'. If you want the contents of
  // 'from_dir' to move under 'to_dir', you will need to specify the contents in
  // the archive config. In the example above, you might need something like:
  //   dirs= ["foo/a", "foo/b"]
  //   rename_dirs = [
  //     { "from_dir":"foo/a", "to_dir":"bar/a" }
  //     { "from_dir":"foo/b", "to_dir":"bar/b" }
  //   ]
  // You can also completely rearrange the directory layout, so that something
  // like this in the build output:
  //   build_out/
  //   |-- bar/
  //   |   `-- resources/
  //   `-- foo/
  //      `-- resources/
  // becomes this layout in the archive:
  //   archive/
  //   `-- resources/
  //       |-- bar/
  //       `-- foo/
  // with:
  //   dirs= ["bar/resources", 'foo/resources']
  //   rename_dirs = [
  //     { "from_dir":"bar/resources", "to_dir":"resources/bar" }
  //     { "from_dir":"foo/resources", "to_dir":"resources/foo" }
  //   ]
  // This may also be used to prefix paths to the entire archive, by renaming
  // the special '.' dir, like:
  //   rename_dirs = [{ "from_dir":".", "to_dir":"new_archive_root" }]
  // NOTE: The '.' dir is the only supported case of 'overlapping' renames.
  // Trying to do something like the following will produce undefined results:
  //   rename_dirs = [
  //     { "from_dir":"bar/do_not_do_this", "to_dir":"do_not_do_this/bar" }
  //     { "from_dir":"do_not_do_this", "to_dir":"really_do_not" }
  //   ]
  repeated ArchiveDirRename rename_dirs = 13;
  // The name of the google cloud storage bucket to upload to.
  // This supports the same string substitutions as |ArchiveData.gcs_path|.
  string gcs_bucket = 3;
  // The destination path in the bucket to upload to. The following string
  // substitutions are built-in:
  // {%position%} -> commit position extracted from got_revision_cp (or
  //     got_src_revision_cp if it is not set).
  // {%commit%} -> commit hash from got_revision.
  // {%timestamp%} -> UTC timestamp in %Y%m%d%H%M%S.
  // {%chromium_version%} -> Chromium version extracted from chrome/VERSION file
  //     in {major}.{minor}.{build}.{patch} format.
  // {%builder_name%} -> name of the builder, fetched from buildbucket
  // {%build_number%} -> number of the current build, fetched from buildbucket
  // {%milestone_position%} -> distance the milestone is from the current canary
  //     milestone, ex: "canary", "canary-1"
  // Custom string substitutions are also supported through the 'custom_vars'
  // parameter in generic_archive().
  string gcs_path = 4;

  enum ArchiveType {
    ARCHIVE_TYPE_UNSPECIFIED = 0;

    // All input files and directories are zipped. gcs_path should be a path
    // whose final component is the name of the archive file.
    // This is the default if nothing is set.
    ARCHIVE_TYPE_ZIP = 1;

    // Upload files unchanged. Directories are not supported. gcs_path should
    // be a path to a directory where the uploaded files will be placed.
    ARCHIVE_TYPE_FILES = 2;

    // All input files and dirs are tar gzipped. gcs_path should be a path
    // whose final component is the name of the archive file.
    ARCHIVE_TYPE_TAR_GZ = 3;

    // All input files and dirs are tar zstded, gcs_path should be a path
    // whose final component is the name of the archive file.
    ARCHIVE_TYPE_TAR_ZSTD = 7;

    // Recursively upload directories and subdirectories. Files are not
    // supported. gcs_path should be a path to the directory where the uploaded
    // items will be placed.
    ARCHIVE_TYPE_RECURSIVE = 4;

    // Upload files flattened such that the file is uploaded directly to the
    // gcs_path directory. ie/ cronet/VERSION file would be uploaded to
    // "gs://{bucket}/test", resulting in {bucket}/test/VERSION. Directories
    // are not supported.
    ARCHIVE_TYPE_FLATTEN_FILES = 5;

    // Similar to ARCHIVE_TYPE_ZIP. But using squashfs tool to compress image.
    // See https://github.com/plougher/squashfs-tools for more information
    // about squashfs format.
    ARCHIVE_TYPE_SQUASHFS = 6;
  }
  ArchiveType archive_type = 5;

  // [Deprecated] Please use squashfs_params instead.
  // Only set this when archive_type is ARCHIVE_TYPE_SQUASHFS.
  // If not set, it will use default compression algorithm gzip.
  // Currently this can only be set to 'zstd'.
  string squashfs_algorithm = 15;

  // Only set this when archive_type is ARCHIVE_TYPE_SQUASHFS.
  SquashfsParams squashfs_params = 16;

  // Only set this when archive_type is ARCHIVE_TYPE_TAR_ZSTD
  TarZstdParams tar_zstd_params = 21;

  // When set, system will update the latest file.
  // Example:
  //   gcs_path = "x86/{%position%}/chrome.zip"
  //   latest_upload['gcs_path'] = "x86/latest/chrome.txt"
  //   latest_upload['gcs_file_content'] = "{%position%}"
  // In this way, the "x86/latest/chrome.txt" will include the latest
  // version info.
  LatestFile latest_upload = 7;

  // Generates a REVISIONS file that inludes the commit hashes from the current
  // revisions.
  // Example:
  //   gcs_path = "x86/{%position%}/chrome.zip"
  //   revisions_file = {
  //     "gcs_path" = "x86/{%position%}/REVISIONS"
  //   }
  RevisionsFile revisions_file = 14;

  // DEPRECATED. Use requires_provenance instead.
  // TODO(morawand): Mark this field as reserved once all archive configs are
  // using requires_provenance.
  string verifiable_key_path = 8;

  // If set to true, generate chain-of-custody provenance for the artifacts
  // listed in this ArchiveData definition and upload them to the GCS path in
  // this config.
  // NOTE: This does not support files archived with ARCHIVE_TYPE_RECURSIVE.
  // Please use one of the other archive types to explicitly list files that
  // need chain-of-custody, or create a zip/tar archive of the dirs, and add the
  // provenance to that.
  bool requires_provenance = 18;

  // If provided, generate a single Software Bill of Materials (SBOM) and
  // upload with the same name as the gcs_path. ARCHIVE_TYPE_ZIP and
  // ARCHIVE_TYPE_FILE are only supported at this time, if the requires_provenance
  // flag is true then the SBOM will be reported as well using the files digest.
  SBOMConfig requires_sbom = 22;

  // A string key to override the default base directory (build dir)
  // for the given file(s), dir(s) for this ArchiveData definition.
  // The provided key is appended to the chromium checkout path.
  // This key is not required and simply defaults to build dir if
  // not provided.
  string base_dir = 9;

  // If sets to true, only upload after tests are run successfully.
  bool only_upload_on_tests_success = 11;

  // The root dir for compression is created by mktmp, which by default has
  // 700 permission. If the consumer needs to read or run the files
  // archived here from a non-root user, we need to overwrite the root dir
  // to 755.
  // Accept any file mode bit in string, e.g. "755" or operators like "g+rwx".
  string root_permission_override = 12;

  // If set to true, the archive step will not fail if one of the referenced
  // directories or files does not exist.
  bool skip_empty_source = 17;

  // If set to true, the '-n' flag will be used for gsutil uploads. This
  // prevents overwriting file content and will skip over existing files.
  bool prevent_overwrites = 19;
}

message Compression {
  // compression_level ([0-9]) - Deflate compression level.
  // (0 - disable, 1 - best speed, 9 - best compression).
  int32 compression_level = 1;
}

message Verification {
  // Maximum time to wait for backend-side package hash verification during
  // package upload. (e.g. '200s', '5m')
  string verification_timeout = 1;
}

// Next id: 8
message CIPDArchiveData {
  // Path to YAML package definition files (relative to build directory).
  repeated string yaml_files = 1;

  // A list of ref names to set for the package instance.
  repeated string refs = 2;

  // A map of tag name -> value to set for the package instance.
  map<string, string> tags = 3;

  // A map of var name -> value to use for vars referenced in package
  // definition file.
  map<string, string> pkg_vars = 4;

  // Sets the compression level.
  Compression compression = 5;

  // Sets time for verification
  Verification verification = 6;

  // If sets to true, add refs to the build only after tests are run
  // successfully.
  bool only_set_refs_on_tests_success = 7;
}

// Next id: 4
message InputProperties {
  repeated ArchiveData archive_datas = 1;
  repeated CIPDArchiveData cipd_archive_datas = 2;

  // The path (relative to checkout_dir) to the file containing
  // specs for archive data. archive_datas and cipd_archive_datas will both
  // be ignored if this path is specfied.
  //
  // An example file looks like
  // {
  //   "cipd_archive_datas": [{
  //     "yaml_files": ["foo",],
  //     "refs": ["{%channel%}",],
  //     "tags": {
  //       "version": "2.3.4.5",
  //     },
  //   },],
  // }
  repeated string source_side_spec_path = 3;

  // Verify the paths specified in the archive_datas without uploading the
  // archive. This option can be used in testing builders to verify the
  // correctness of build results and the archive config.
  // This option could be enabled via archive input properties with
  // source_side_spec_path specified.
  bool verify_paths_only = 4;
}

// Next id: 3
message SBOMConfig {
  // Build the SBOM based on the supplied gn_targets.
  repeated string gn_targets = 1;
  // Optionally upload the generated artifacts including
  // the SBOM to this folder. The SBOM will still be
  // uploaded with the archived artifact in the archiveData.gcs_path.
  // This should be a fully qualified gcs path, such as
  // 'gs://bucket/folder'.
  string gcs_folder = 2;
}