testing/merge_scripts/standard_gtest_merge_test.py

#!/usr/bin/env vpython3
# Copyright 2014 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import json
import logging
import os
import shutil
import sys
import tempfile
import unittest
from unittest import mock

import six

import common_merge_script_tests
import standard_gtest_merge

# gtest json output for successfully finished shard #0.
GOOD_GTEST_JSON_0 = {
    'all_tests': [
        'AlignedMemoryTest.DynamicAllocation',
        'AlignedMemoryTest.ScopedDynamicAllocation',
        'AlignedMemoryTest.StackAlignment',
        'AlignedMemoryTest.StaticAlignment',
    ],
    'disabled_tests': [
        'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
        'FileTest.TouchGetInfo',
        'MessageLoopTestTypeDefault.EnsureDeletion',
    ],
    'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
    'per_iteration_data': [{
        'AlignedMemoryTest.DynamicAllocation': [{
            'elapsed_time_ms': 0,
            'losless_snippet': True,
            'output_snippet': 'blah\\n',
            'output_snippet_base64': 'YmxhaAo=',
            'status': 'SUCCESS',
        }],
        'AlignedMemoryTest.ScopedDynamicAllocation': [{
            'elapsed_time_ms': 0,
            'losless_snippet': True,
            'output_snippet': 'blah\\n',
            'output_snippet_base64': 'YmxhaAo=',
            'status': 'SUCCESS',
        }],
    }],
    'test_locations': {
        'AlignedMemoryTest.DynamicAllocation': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 123,
        },
        'AlignedMemoryTest.ScopedDynamicAllocation': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 456,
        },
        # This is a test from a different shard, but this happens in practice
        # and we should not fail if information is repeated.
        'AlignedMemoryTest.StaticAlignment': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 12,
        },
    },
}

# gtest json output for successfully finished shard #1.
GOOD_GTEST_JSON_1 = {
    'all_tests': [
        'AlignedMemoryTest.DynamicAllocation',
        'AlignedMemoryTest.ScopedDynamicAllocation',
        'AlignedMemoryTest.StackAlignment',
        'AlignedMemoryTest.StaticAlignment',
    ],
    'disabled_tests': [
        'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
        'FileTest.TouchGetInfo',
        'MessageLoopTestTypeDefault.EnsureDeletion',
    ],
    'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
    'per_iteration_data': [{
        'AlignedMemoryTest.StackAlignment': [{
            'elapsed_time_ms': 0,
            'losless_snippet': True,
            'output_snippet': 'blah\\n',
            'output_snippet_base64': 'YmxhaAo=',
            'status': 'SUCCESS',
        }],
        'AlignedMemoryTest.StaticAlignment': [{
            'elapsed_time_ms': 0,
            'losless_snippet': True,
            'output_snippet': 'blah\\n',
            'output_snippet_base64': 'YmxhaAo=',
            'status': 'SUCCESS',
        }],
    }],
    'test_locations': {
        'AlignedMemoryTest.StackAlignment': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 789,
        },
        'AlignedMemoryTest.StaticAlignment': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 12,
        },
    },
}

TIMED_OUT_GTEST_JSON_1 = {
    'disabled_tests': [],
    'global_tags': [],
    'all_tests': [
        'AlignedMemoryTest.DynamicAllocation',
        'AlignedMemoryTest.ScopedDynamicAllocation',
        'AlignedMemoryTest.StackAlignment',
        'AlignedMemoryTest.StaticAlignment',
    ],
    'per_iteration_data': [{
        'AlignedMemoryTest.StackAlignment': [{
            'elapsed_time_ms': 54000,
            'losless_snippet': True,
            'output_snippet': 'timed out',
            'output_snippet_base64': '',
            'status': 'FAILURE',
        }],
        'AlignedMemoryTest.StaticAlignment': [{
            'elapsed_time_ms': 0,
            'losless_snippet': True,
            'output_snippet': '',
            'output_snippet_base64': '',
            'status': 'NOTRUN',
        }],
    }],
    'test_locations': {
        'AlignedMemoryTest.StackAlignment': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 789,
        },
        'AlignedMemoryTest.StaticAlignment': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 12,
        },
    },
}

# GOOD_GTEST_JSON_0 and GOOD_GTEST_JSON_1 merged.
GOOD_GTEST_JSON_MERGED = {
    'all_tests': [
        'AlignedMemoryTest.DynamicAllocation',
        'AlignedMemoryTest.ScopedDynamicAllocation',
        'AlignedMemoryTest.StackAlignment',
        'AlignedMemoryTest.StaticAlignment',
    ],
    'disabled_tests': [
        'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
        'FileTest.TouchGetInfo',
        'MessageLoopTestTypeDefault.EnsureDeletion',
    ],
    'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
    'missing_shards': [],
    'per_iteration_data': [{
        'AlignedMemoryTest.DynamicAllocation': [{
            'elapsed_time_ms': 0,
            'losless_snippet': True,
            'output_snippet': 'blah\\n',
            'output_snippet_base64': 'YmxhaAo=',
            'status': 'SUCCESS',
        }],
        'AlignedMemoryTest.ScopedDynamicAllocation': [{
            'elapsed_time_ms': 0,
            'losless_snippet': True,
            'output_snippet': 'blah\\n',
            'output_snippet_base64': 'YmxhaAo=',
            'status': 'SUCCESS',
        }],
        'AlignedMemoryTest.StackAlignment': [{
            'elapsed_time_ms': 0,
            'losless_snippet': True,
            'output_snippet': 'blah\\n',
            'output_snippet_base64': 'YmxhaAo=',
            'status': 'SUCCESS',
        }],
        'AlignedMemoryTest.StaticAlignment': [{
            'elapsed_time_ms': 0,
            'losless_snippet': True,
            'output_snippet': 'blah\\n',
            'output_snippet_base64': 'YmxhaAo=',
            'status': 'SUCCESS',
        }],
    }],
    'swarming_summary': {
        u'shards': [{
            u'state': u'COMPLETED',
            u'outputs_ref': {
                u'view_url': u'blah',
            },
        }],
    },
    'test_locations': {
        'AlignedMemoryTest.StackAlignment': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 789,
        },
        'AlignedMemoryTest.StaticAlignment': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 12,
        },
        'AlignedMemoryTest.DynamicAllocation': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 123,
        },
        'AlignedMemoryTest.ScopedDynamicAllocation': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 456,
        },
    },
}

# Only shard #1 finished. UNRELIABLE_RESULTS is set.
BAD_GTEST_JSON_ONLY_1_SHARD = {
    'all_tests': [
        'AlignedMemoryTest.DynamicAllocation',
        'AlignedMemoryTest.ScopedDynamicAllocation',
        'AlignedMemoryTest.StackAlignment',
        'AlignedMemoryTest.StaticAlignment',
    ],
    'disabled_tests': [
        'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
        'FileTest.TouchGetInfo',
        'MessageLoopTestTypeDefault.EnsureDeletion',
    ],
    'global_tags': [
        'CPU_64_BITS',
        'MODE_DEBUG',
        'OS_LINUX',
        'OS_POSIX',
        'UNRELIABLE_RESULTS',
    ],
    'missing_shards': [0],
    'per_iteration_data': [{
        'AlignedMemoryTest.StackAlignment': [{
            'elapsed_time_ms': 0,
            'losless_snippet': True,
            'output_snippet': 'blah\\n',
            'output_snippet_base64': 'YmxhaAo=',
            'status': 'SUCCESS',
        }],
        'AlignedMemoryTest.StaticAlignment': [{
            'elapsed_time_ms': 0,
            'losless_snippet': True,
            'output_snippet': 'blah\\n',
            'output_snippet_base64': 'YmxhaAo=',
            'status': 'SUCCESS',
        }],
    }],
    'test_locations': {
        'AlignedMemoryTest.StackAlignment': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 789,
        },
        'AlignedMemoryTest.StaticAlignment': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 12,
        },
    },
}

# GOOD_GTEST_JSON_0 and TIMED_OUT_GTEST_JSON_1 merged.
TIMED_OUT_GTEST_JSON_MERGED = {
    'all_tests': [
        'AlignedMemoryTest.DynamicAllocation',
        'AlignedMemoryTest.ScopedDynamicAllocation',
        'AlignedMemoryTest.StackAlignment',
        'AlignedMemoryTest.StaticAlignment',
    ],
    'disabled_tests': [
        'ConditionVariableTest.TimeoutAcrossSetTimeOfDay',
        'FileTest.TouchGetInfo',
        'MessageLoopTestTypeDefault.EnsureDeletion',
    ],
    'global_tags': ['CPU_64_BITS', 'MODE_DEBUG', 'OS_LINUX', 'OS_POSIX'],
    'missing_shards': [],
    'per_iteration_data': [{
        'AlignedMemoryTest.DynamicAllocation': [{
            'elapsed_time_ms': 0,
            'losless_snippet': True,
            'output_snippet': 'blah\\n',
            'output_snippet_base64': 'YmxhaAo=',
            'status': 'SUCCESS',
        }],
        'AlignedMemoryTest.ScopedDynamicAllocation': [{
            'elapsed_time_ms': 0,
            'losless_snippet': True,
            'output_snippet': 'blah\\n',
            'output_snippet_base64': 'YmxhaAo=',
            'status': 'SUCCESS',
        }],
        'AlignedMemoryTest.StackAlignment': [{
            'elapsed_time_ms': 54000,
            'losless_snippet': True,
            'output_snippet': 'timed out',
            'output_snippet_base64': '',
            'status': 'FAILURE',
        }],
        'AlignedMemoryTest.StaticAlignment': [{
            'elapsed_time_ms': 0,
            'losless_snippet': True,
            'output_snippet': '',
            'output_snippet_base64': '',
            'status': 'NOTRUN',
        }],
    }],
    'swarming_summary': {
        u'shards': [
            {
                u'state': u'COMPLETED',
            },
            {
                u'state': u'TIMED_OUT',
            },
        ],
    },
    'test_locations': {
        'AlignedMemoryTest.StackAlignment': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 789,
        },
        'AlignedMemoryTest.StaticAlignment': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 12,
        },
        'AlignedMemoryTest.DynamicAllocation': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 123,
        },
        'AlignedMemoryTest.ScopedDynamicAllocation': {
            'file': 'foo/bar/allocation_test.cc',
            'line': 456,
        },
    },
}


class _StandardGtestMergeTest(unittest.TestCase):

  def setUp(self):
    self.temp_dir = tempfile.mkdtemp()

  def tearDown(self):
    shutil.rmtree(self.temp_dir)

  def _write_temp_file(self, path, content):
    abs_path = os.path.join(self.temp_dir, path.replace('/', os.sep))
    if not os.path.exists(os.path.dirname(abs_path)):
      os.makedirs(os.path.dirname(abs_path))
    with open(abs_path, 'w') as f:
      if isinstance(content, dict):
        json.dump(content, f)
      else:
        assert isinstance(content, str)
        f.write(content)
    return abs_path


class LoadShardJsonTest(_StandardGtestMergeTest):

  def test_double_digit_jsons(self):
    jsons_to_merge = []
    for i in range(15):
      json_dir = os.path.join(self.temp_dir, str(i))
      json_path = os.path.join(json_dir, 'output.json')
      if not os.path.exists(json_dir):
        os.makedirs(json_dir)
      with open(json_path, 'w') as f:
        json.dump({'all_tests': ['LoadShardJsonTest.test%d' % i]}, f)
      jsons_to_merge.append(json_path)

    content, err = standard_gtest_merge.load_shard_json(0, None, jsons_to_merge)
    self.assertEqual({'all_tests': ['LoadShardJsonTest.test0']}, content)
    self.assertIsNone(err)

    content, err = standard_gtest_merge.load_shard_json(12, None,
                                                        jsons_to_merge)
    self.assertEqual({'all_tests': ['LoadShardJsonTest.test12']}, content)
    self.assertIsNone(err)

  def test_double_task_id_jsons(self):
    jsons_to_merge = []
    for i in range(15):
      json_dir = os.path.join(self.temp_dir, 'deadbeef%d' % i)
      json_path = os.path.join(json_dir, 'output.json')
      if not os.path.exists(json_dir):
        os.makedirs(json_dir)
      with open(json_path, 'w') as f:
        json.dump({'all_tests': ['LoadShardJsonTest.test%d' % i]}, f)
      jsons_to_merge.append(json_path)

    content, err = standard_gtest_merge.load_shard_json(0, 'deadbeef0',
                                                        jsons_to_merge)
    self.assertEqual({'all_tests': ['LoadShardJsonTest.test0']}, content)
    self.assertIsNone(err)

    content, err = standard_gtest_merge.load_shard_json(12, 'deadbeef12',
                                                        jsons_to_merge)
    self.assertEqual({'all_tests': ['LoadShardJsonTest.test12']}, content)
    self.assertIsNone(err)


class MergeShardResultsTest(_StandardGtestMergeTest):
  """Tests for merge_shard_results function."""

  # pylint: disable=super-with-arguments
  def setUp(self):
    super(MergeShardResultsTest, self).setUp()
    self.summary = None
    self.test_files = []

  # pylint: enable=super-with-arguments

  def stage(self, summary, files):
    self.summary = self._write_temp_file('summary.json', summary)
    for path, content in files.items():
      abs_path = self._write_temp_file(path, content)
      self.test_files.append(abs_path)

  def call(self):
    stdout = six.StringIO()
    with mock.patch('sys.stdout', stdout):
      merged = standard_gtest_merge.merge_shard_results(self.summary,
                                                        self.test_files)
      return merged, stdout.getvalue().strip()

  def assertUnicodeEquals(self, expectation, result):

    def convert_to_unicode(key_or_value):
      if isinstance(key_or_value, str):
        return six.text_type(key_or_value)
      if isinstance(key_or_value, dict):
        return {
            convert_to_unicode(k): convert_to_unicode(v)
            for k, v in key_or_value.items()
        }
      if isinstance(key_or_value, list):
        return [convert_to_unicode(x) for x in key_or_value]
      return key_or_value

    unicode_expectations = convert_to_unicode(expectation)
    unicode_result = convert_to_unicode(result)
    self.assertEqual(unicode_expectations, unicode_result)

  def test_ok(self):
    # Two shards, both successfully finished.
    self.stage(
        {
            u'shards': [
                {
                    u'state': u'COMPLETED',
                },
                {
                    u'state': u'COMPLETED',
                },
            ],
        }, {
            '0/output.json': GOOD_GTEST_JSON_0,
            '1/output.json': GOOD_GTEST_JSON_1,
        })
    merged, stdout = self.call()
    merged['swarming_summary'] = {
        'shards': [{
            u'state': u'COMPLETED',
            u'outputs_ref': {
                u'view_url': u'blah',
            },
        }],
    }
    self.assertUnicodeEquals(GOOD_GTEST_JSON_MERGED, merged)
    self.assertEqual('', stdout)

  def test_timed_out(self):
    # Two shards, both successfully finished.
    self.stage(
        {
            'shards': [
                {
                    'state': 'COMPLETED',
                },
                {
                    'state': 'TIMED_OUT',
                },
            ],
        }, {
            '0/output.json': GOOD_GTEST_JSON_0,
            '1/output.json': TIMED_OUT_GTEST_JSON_1,
        })
    merged, stdout = self.call()

    self.assertUnicodeEquals(TIMED_OUT_GTEST_JSON_MERGED, merged)
    self.assertIn('Test runtime exceeded allocated time\n', stdout)

  def test_missing_summary_json(self):
    # summary.json is missing, should return None and emit warning.
    self.summary = os.path.join(self.temp_dir, 'summary.json')
    merged, output = self.call()
    self.assertEqual(None, merged)
    self.assertIn('@@@STEP_WARNINGS@@@', output)
    self.assertIn('summary.json is missing or can not be read', output)

  def test_unfinished_shards(self):
    # Only one shard (#1) finished. Shard #0 did not.
    self.stage({
        u'shards': [
            None,
            {
                u'state': u'COMPLETED',
            },
        ],
    }, {
        u'1/output.json': GOOD_GTEST_JSON_1,
    })
    merged, stdout = self.call()
    merged.pop('swarming_summary')
    self.assertUnicodeEquals(BAD_GTEST_JSON_ONLY_1_SHARD, merged)
    self.assertIn('@@@STEP_WARNINGS@@@\nsome shards did not complete: 0\n',
                  stdout)
    self.assertIn(
        '@@@STEP_LOG_LINE@some shards did not complete: 0@'
        'Missing results from the following shard(s): 0@@@\n', stdout)

  def test_missing_output_json(self):
    # Shard #0 output json is missing.
    self.stage(
        {
            u'shards': [
                {
                    u'state': u'COMPLETED',
                },
                {
                    u'state': u'COMPLETED',
                },
            ],
        }, {
            u'1/output.json': GOOD_GTEST_JSON_1,
        })
    merged, stdout = self.call()
    merged.pop('swarming_summary')
    self.assertUnicodeEquals(BAD_GTEST_JSON_ONLY_1_SHARD, merged)
    self.assertIn('No result was found: '
                  'shard 0 test output was missing', stdout)

  def test_large_output_json(self):
    # a shard is too large.
    self.stage(
        {
            u'shards': [
                {
                    u'state': u'COMPLETED',
                },
                {
                    u'state': u'COMPLETED',
                },
            ],
        }, {
            '0/output.json': GOOD_GTEST_JSON_0,
            '1/output.json': GOOD_GTEST_JSON_1,
        })
    old_json_limit = standard_gtest_merge.OUTPUT_JSON_SIZE_LIMIT
    len0 = len(json.dumps(GOOD_GTEST_JSON_0))
    len1 = len(json.dumps(GOOD_GTEST_JSON_1))
    large_shard = '0' if len0 > len1 else '1'
    try:
      # Override max output.json size just for this test.
      standard_gtest_merge.OUTPUT_JSON_SIZE_LIMIT = min(len0, len1)
      merged, stdout = self.call()
      merged.pop('swarming_summary')
      self.assertUnicodeEquals(BAD_GTEST_JSON_ONLY_1_SHARD, merged)
      self.assertIn(
          'No result was found: '
          'shard %s test output exceeded the size limit' % large_shard, stdout)
    finally:
      standard_gtest_merge.OUTPUT_JSON_SIZE_LIMIT = old_json_limit


class CommandLineTest(common_merge_script_tests.CommandLineTest):

  # pylint: disable=super-with-arguments
  def __init__(self, methodName='runTest'):
    super(CommandLineTest, self).__init__(methodName, standard_gtest_merge)

  # pylint: enable=super-with-arguments


if __name__ == '__main__':
  logging.basicConfig(
      level=logging.DEBUG if '-v' in sys.argv else logging.ERROR)
  if '-v' in sys.argv:
    unittest.TestCase.maxDiff = None
  unittest.main()