Merge branch 'main' into panel-references

Author: cesco-f

.buildkite/ftr_platform_stateful_configs.yml

@@ -138,6 +138,7 @@ enabled:
   - x-pack/platform/test/localization/config.ja_jp.ts
   - x-pack/platform/test/localization/config.fr_fr.ts
   - x-pack/platform/test/localization/config.zh_cn.ts
+  - x-pack/platform/test/localization/config.de_de.ts
   - x-pack/platform/test/alerting_api_integration/basic/config.ts
   - x-pack/platform/test/alerting_api_integration/security_and_spaces/group1/config.ts
   - x-pack/platform/test/alerting_api_integration/security_and_spaces/group2/config.ts

.buildkite/ftr_security_serverless_configs.yml

@@ -19,6 +19,10 @@ disabled:
   # MKI only configs files
   - x-pack/test_serverless/functional/test_suites/security/config.mki_only.ts
 
+  # Detection Rules Management base configs
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/configs/serverless/rules_management.essentials.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/configs/serverless/rules_management.complete.config.ts
+
 defaultQueue: 'n2-4-spot'
 enabled:
   - x-pack/test_serverless/api_integration/test_suites/security/config.ts

.buildkite/ftr_security_stateful_configs.yml

@@ -33,6 +33,10 @@ disabled:
   # Gen AI Evals run weekly via their own pipeline
   - x-pack/test/security_solution_api_integration/test_suites/genai/evaluations/trial_license_complete_tier/configs/ess.config.ts
 
+  # Detection Rules Management base configs
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/configs/ess/rules_management.basic.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/rules_management/configs/ess/rules_management.trial.config.ts
+
 defaultQueue: 'n2-4-spot'
 enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/actions/trial_license_complete_tier/configs/ess.config.ts

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -463,6 +463,9 @@ export async function pickTestGroupRunOrder() {
               ...expandAgentQueue('n2-4-spot'),
               diskSizeGb: 85,
             },
+            env: {
+              SCOUT_TARGET_TYPE: 'local',
+            },
             retry: {
               automatic: [
                 { exit_status: '-1', limit: 3 },
@@ -481,6 +484,9 @@ export async function pickTestGroupRunOrder() {
             timeout_in_minutes: 120,
             key: 'jest-integration',
             agents: expandAgentQueue('n2-4-spot'),
+            env: {
+              SCOUT_TARGET_TYPE: 'local',
+            },
             retry: {
               automatic: [
                 { exit_status: '-1', limit: 3 },
@@ -516,6 +522,7 @@ export async function pickTestGroupRunOrder() {
                   timeout_in_minutes: 90,
                   agents: expandAgentQueue(queue),
                   env: {
+                    SCOUT_TARGET_TYPE: 'local',
                     FTR_CONFIG_GROUP_KEY: key,
                     ...ftrExtraArgs,
                     ...envFromlabels,

.buildkite/scripts/lifecycle/post_command.sh

@@ -29,6 +29,12 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'src/platform/test/**/screenshots/failure/*.png'
   buildkite-agent artifact upload 'src/platform/test/**/screenshots/session/*.png'
   buildkite-agent artifact upload 'src/platform/test/functional/failure_debug/html/*.html'
+  buildkite-agent artifact upload 'x-pack/platform/test/**/screenshots/diff/*.png'
+  buildkite-agent artifact upload 'x-pack/platform/test/**/screenshots/failure/*.png'
+  buildkite-agent artifact upload 'x-pack/platform/test/**/screenshots/session/*.png'
+  buildkite-agent artifact upload 'x-pack/platform/test_serverless/**/screenshots/diff/*.png'
+  buildkite-agent artifact upload 'x-pack/platform/test_serverless/**/screenshots/failure/*.png'
+  buildkite-agent artifact upload 'x-pack/platform/test_serverless/**/screenshots/session/*.png'
   buildkite-agent artifact upload 'x-pack/test/**/screenshots/diff/*.png'
   buildkite-agent artifact upload 'x-pack/test/**/screenshots/failure/*.png'
   buildkite-agent artifact upload 'x-pack/test/**/screenshots/session/*.png'

.buildkite/scripts/steps/checks/dependencies_diff.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo '--- Security: check 3rd-party dependencies'
+
+if is_pr && ! is_auto_commit_disabled; then
+  ts-node .buildkite/scripts/steps/security/dependencies_diff.ts
+fi
+
+check_for_changed_files "security: 3rd-party dependencies" true
+
+echo "security: check 3rd-party dependencies ✅"

.buildkite/scripts/steps/checks/quick_checks.txt

@@ -22,3 +22,4 @@
 .buildkite/scripts/steps/checks/dependencies_missing_owner.sh
 .buildkite/scripts/steps/checks/validate_pipelines.sh
 .buildkite/scripts/steps/checks/check_scout_config.sh
+.buildkite/scripts/steps/checks/dependencies_diff.sh

.buildkite/scripts/steps/security/dependencies_diff.ts

@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { execSync } from 'child_process';
+import { appendFile, readFile, writeFile } from 'fs/promises';
+import { join } from 'path';
+import { getGithubClient } from '#pipeline-utils';
+
+const octokit = getGithubClient();
+
+const INTERNAL_PACKAGE_PREFIX_REGEX = /^(@kbn|@elastic)\//;
+
+async function getFileContent(path: string, ref: string) {
+  console.info(`Fetching content for ${path} at ref ${ref}`);
+
+  const response = await octokit.repos.getContent({
+    owner: process.env.GITHUB_PR_BASE_OWNER!,
+    repo: process.env.GITHUB_PR_BASE_REPO!,
+    path,
+    ref,
+  });
+
+  if (Array.isArray(response.data) || response.data.type !== 'file') {
+    throw new Error(`Expected file at ${path}`);
+  }
+
+  // @ts-ignore ts-node doesn't infer the type of response.data.content
+  const content = Buffer.from(response.data.content, 'base64').toString('utf8');
+
+  return JSON.parse(content);
+}
+
+async function getDependenciesDiff() {
+  const oldPackageJson = await getFileContent('package.json', process.env.GITHUB_PR_MERGE_BASE!);
+  const headSha = execSync('git rev-parse HEAD').toString().trim();
+  const newPackageJson = await getFileContent('package.json', headSha);
+
+  const oldDeps = { ...oldPackageJson.dependencies, ...oldPackageJson.devDependencies };
+  const newDeps = { ...newPackageJson.dependencies, ...newPackageJson.devDependencies };
+
+  const newPackages = [];
+  const removedPackages = [];
+
+  console.info('Comparing dependencies...');
+
+  for (const name in newDeps) {
+    if (!oldDeps[name] && !name.match(INTERNAL_PACKAGE_PREFIX_REGEX)) {
+      newPackages.push(name);
+    }
+  }
+
+  for (const name in oldDeps) {
+    if (!newDeps[name] && !name.match(INTERNAL_PACKAGE_PREFIX_REGEX)) {
+      removedPackages.push(name);
+    }
+  }
+
+  return { added: newPackages, removed: removedPackages };
+}
+
+async function main() {
+  // Skipping PRs from Renovate
+  if (process.env.GIT_BRANCH?.startsWith('renovate')) {
+    return;
+  }
+
+  const changedFiles = execSync(
+    `git diff --name-only --diff-filter=M ${process.env.GITHUB_PR_MERGE_BASE} HEAD`
+  )
+    .toString()
+    .trim()
+    .split('\n');
+
+  const packageJsonChanged = changedFiles.some((file) => file === 'package.json');
+  const dependenciesAdded = changedFiles.some((file) => file.match('third_party_packages.txt'));
+  const filePath = join(__dirname, 'third_party_packages.txt');
+
+  // Reverting changes (if dependency was added, then removed in the same PR)
+  if (!packageJsonChanged && dependenciesAdded) {
+    console.info(`Reverting changes for ${filePath}`);
+
+    try {
+      execSync(`git cat-file -e ${process.env.GITHUB_PR_MERGE_BASE}:${filePath}`, {
+        stdio: 'ignore',
+      });
+    } catch (error) {
+      console.info(`File does not exist in merge base, skipping revert`);
+      return;
+    }
+
+    execSync(`git checkout ${process.env.GITHUB_PR_MERGE_BASE} -- ${filePath}`);
+
+    return;
+  }
+
+  const { added, removed } = await getDependenciesDiff();
+
+  if (!added.length && !removed.length) {
+    console.info('No third party packages added or removed');
+    return;
+  }
+
+  const existingContent = await readFile(filePath, 'utf8');
+
+  const existingLines = new Set(existingContent.split('\n').filter(Boolean));
+
+  if (added.length && !removed.length) {
+    const newEntries = added.filter((entry) => !existingLines.has(entry));
+
+    if (newEntries.length) {
+      const data = newEntries.join('\n') + '\n';
+      await appendFile(filePath, data);
+    }
+
+    return;
+  }
+
+  const updatedLines = [...new Set([...added, ...existingLines])].filter(
+    (line) => !removed.includes(line)
+  );
+
+  await writeFile(filePath, updatedLines.join('\n') + '\n');
+}
+
+main();

.buildkite/scripts/steps/security/third_party_packages.txt

@@ -0,0 +1 @@
+@langchain/langgraph-checkpoint