Switch monkey test to platform mode.
BUG=663127
Review-Url: https://codereview.chromium.org/2505713002
Cr-Commit-Position: refs/heads/master@{#432695}
diff --git a/build/android/pylib/base/test_instance_factory.py b/build/android/pylib/base/test_instance_factory.py
index 129bd7f..20b99d63 100644
--- a/build/android/pylib/base/test_instance_factory.py
+++ b/build/android/pylib/base/test_instance_factory.py
@@ -5,6 +5,7 @@
from pylib.gtest import gtest_test_instance
from pylib.instrumentation import instrumentation_test_instance
from pylib.junit import junit_test_instance
+from pylib.monkey import monkey_test_instance
from pylib.perf import perf_test_instance
from pylib.utils import isolator
@@ -19,6 +20,8 @@
args, isolator.Isolator(), error_func)
elif args.command == 'junit':
return junit_test_instance.JunitTestInstance(args, error_func)
+ elif args.command == 'monkey':
+ return monkey_test_instance.MonkeyTestInstance(args, error_func)
elif args.command == 'perf':
return perf_test_instance.PerfTestInstance(args, error_func)
diff --git a/build/android/pylib/base/test_run_factory.py b/build/android/pylib/base/test_run_factory.py
index d706fd4..0d71e97 100644
--- a/build/android/pylib/base/test_run_factory.py
+++ b/build/android/pylib/base/test_run_factory.py
@@ -5,9 +5,11 @@
from pylib.gtest import gtest_test_instance
from pylib.instrumentation import instrumentation_test_instance
from pylib.junit import junit_test_instance
+from pylib.monkey import monkey_test_instance
from pylib.local.device import local_device_environment
from pylib.local.device import local_device_gtest_run
from pylib.local.device import local_device_instrumentation_test_run
+from pylib.local.device import local_device_monkey_test_run
from pylib.local.device import local_device_perf_test_run
from pylib.local.machine import local_machine_environment
from pylib.local.machine import local_machine_junit_test_run
@@ -33,6 +35,9 @@
instrumentation_test_instance.InstrumentationTestInstance):
return (local_device_instrumentation_test_run
.LocalDeviceInstrumentationTestRun(env, test_instance))
+ if isinstance(test_instance, monkey_test_instance.MonkeyTestInstance):
+ return (local_device_monkey_test_run
+ .LocalDeviceMonkeyTestRun(env, test_instance))
if isinstance(test_instance,
perf_test_instance.PerfTestInstance):
return _CreatePerfTestRun(args, env, test_instance)
diff --git a/build/android/pylib/local/device/local_device_instrumentation_test_run.py b/build/android/pylib/local/device/local_device_instrumentation_test_run.py
index 2705289f..7398f62 100644
--- a/build/android/pylib/local/device/local_device_instrumentation_test_run.py
+++ b/build/android/pylib/local/device/local_device_instrumentation_test_run.py
@@ -60,6 +60,7 @@
super(LocalDeviceInstrumentationTestRun, self).__init__(env, test_instance)
self._flag_changers = {}
+ #override
def TestPackage(self):
return self._test_instance.suite
diff --git a/build/android/pylib/local/device/local_device_monkey_test_run.py b/build/android/pylib/local/device/local_device_monkey_test_run.py
new file mode 100644
index 0000000..8f293c6
--- /dev/null
+++ b/build/android/pylib/local/device/local_device_monkey_test_run.py
@@ -0,0 +1,126 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+
+from devil.android import device_errors
+from devil.android.sdk import intent
+from pylib import constants
+from pylib.base import base_test_result
+from pylib.local.device import local_device_test_run
+
+
+_CHROME_PACKAGE = constants.PACKAGE_INFO['chrome'].package
+
+class LocalDeviceMonkeyTestRun(local_device_test_run.LocalDeviceTestRun):
+ def __init__(self, env, test_instance):
+ super(LocalDeviceMonkeyTestRun, self).__init__(env, test_instance)
+
+ def TestPackage(self):
+ return 'monkey'
+
+ #override
+ def SetUp(self):
+ pass
+
+ #override
+ def _RunTest(self, device, test):
+ device.ClearApplicationState(self._test_instance.package)
+
+ # Chrome crashes are not always caught by Monkey test runner.
+ # Launch Chrome and verify Chrome has the same PID before and after
+ # the test.
+ device.StartActivity(
+ intent.Intent(package=self._test_instance.package,
+ activity=self._test_instance.activity,
+ action='android.intent.action.MAIN'),
+ blocking=True, force_stop=True)
+ before_pids = device.GetPids(self._test_instance.package)
+
+ output = ''
+ if before_pids:
+ if len(before_pids.get(self._test_instance.package, [])) > 1:
+ raise Exception(
+ 'At most one instance of process %s expected but found pids: '
+ '%s' % (self._test_instance.package, before_pids))
+ output = '\n'.join(self._LaunchMonkeyTest(device))
+ after_pids = device.GetPids(self._test_instance.package)
+
+ crashed = True
+ if not self._test_instance.package in before_pids:
+ logging.error('Failed to start the process.')
+ elif not self._test_instance.package in after_pids:
+ logging.error('Process %s has died.',
+ before_pids[self._test_instance.package])
+ elif (before_pids[self._test_instance.package] !=
+ after_pids[self._test_instance.package]):
+ logging.error('Detected process restart %s -> %s',
+ before_pids[self._test_instance.package],
+ after_pids[self._test_instance.package])
+ else:
+ crashed = False
+
+ success_pattern = 'Events injected: %d' % self._test_instance.event_count
+ if success_pattern in output and not crashed:
+ result = base_test_result.BaseTestResult(
+ test, base_test_result.ResultType.PASS, log=output)
+ else:
+ result = base_test_result.BaseTestResult(
+ test, base_test_result.ResultType.FAIL, log=output)
+ if 'chrome' in self._test_instance.package:
+ logging.warning('Starting MinidumpUploadService...')
+ # TODO(jbudorick): Update this after upstreaming.
+ minidump_intent = intent.Intent(
+ action='%s.crash.ACTION_FIND_ALL' % _CHROME_PACKAGE,
+ package=self._test_instance.package,
+ activity='%s.crash.MinidumpUploadService' % _CHROME_PACKAGE)
+ try:
+ device.RunShellCommand(
+ ['am', 'startservice'] + minidump_intent.am_args,
+ as_root=True, check_return=True)
+ except device_errors.CommandFailedError:
+ logging.exception('Failed to start MinidumpUploadService')
+
+ return result
+
+ #override
+ def TearDown(self):
+ pass
+
+ #override
+ def _CreateShards(self, tests):
+ return tests
+
+ #override
+ def _ShouldShard(self):
+ # TODO(mikecase): Run Monkey test concurrently on each attached device.
+ return False
+
+ #override
+ def _GetTests(self):
+ return ['MonkeyTest']
+
+ def _LaunchMonkeyTest(self, device):
+ try:
+ cmd = ['monkey',
+ '-p', self._test_instance.package,
+ '--throttle', str(self._test_instance.throttle),
+ '-s', str(self._test_instance.seed),
+ '--monitor-native-crashes',
+ '--kill-process-after-error']
+ for category in self._test_instance.categories:
+ cmd.extend(['-c', category])
+ for _ in range(self._test_instance.verbose_count):
+ cmd.append('-v')
+ cmd.append(str(self._test_instance.event_count))
+ return device.RunShellCommand(
+ cmd, timeout=self._test_instance.timeout)
+ finally:
+ try:
+ # Kill the monkey test process on the device. If you manually
+ # interupt the test run, this will prevent the monkey test from
+ # continuing to run.
+ device.KillAll('com.android.commands.monkey')
+ except device_errors.CommandFailedError:
+ pass
diff --git a/build/android/pylib/monkey/monkey_test_instance.py b/build/android/pylib/monkey/monkey_test_instance.py
new file mode 100644
index 0000000..4200ecd3
--- /dev/null
+++ b/build/android/pylib/monkey/monkey_test_instance.py
@@ -0,0 +1,72 @@
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import random
+
+from pylib import constants
+from pylib.base import test_instance
+
+
+_SINGLE_EVENT_TIMEOUT = 100 # Milliseconds
+
+class MonkeyTestInstance(test_instance.TestInstance):
+
+ def __init__(self, args, _):
+ super(MonkeyTestInstance, self).__init__()
+
+ self._categories = args.categories
+ self._event_count = args.event_count
+ self._seed = args.seed or random.randint(1, 100)
+ self._throttle = args.throttle
+ self._verbose_count = args.verbose_count
+
+ self._package = constants.PACKAGE_INFO[args.browser].package
+ self._activity = constants.PACKAGE_INFO[args.browser].activity
+
+ self._timeout_ms = (self.event_count *
+ (self.throttle + _SINGLE_EVENT_TIMEOUT))
+
+ #override
+ def TestType(self):
+ return 'monkey'
+
+ #override
+ def SetUp(self):
+ pass
+
+ #override
+ def TearDown(self):
+ pass
+
+ @property
+ def activity(self):
+ return self._activity
+
+ @property
+ def categories(self):
+ return self._categories
+
+ @property
+ def event_count(self):
+ return self._event_count
+
+ @property
+ def package(self):
+ return self._package
+
+ @property
+ def seed(self):
+ return self._seed
+
+ @property
+ def throttle(self):
+ return self._throttle
+
+ @property
+ def timeout(self):
+ return self._timeout_ms
+
+ @property
+ def verbose_count(self):
+ return self._verbose_count
diff --git a/build/android/pylib/monkey/setup.py b/build/android/pylib/monkey/setup.py
deleted file mode 100644
index fe690a5..0000000
--- a/build/android/pylib/monkey/setup.py
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Generates test runner factory and tests for monkey tests."""
-
-from pylib.monkey import test_runner
-
-
-def Setup(test_options):
- """Create and return the test runner factory and tests.
-
- Args:
- test_options: A MonkeyOptions object.
-
- Returns:
- A tuple of (TestRunnerFactory, tests).
- """
- # Token to replicate across devices as the "test". The TestRunner does all of
- # the work to run the test.
- tests = ['MonkeyTest']
-
- def TestRunnerFactory(device, shard_index):
- return test_runner.TestRunner(
- test_options, device, shard_index)
-
- return (TestRunnerFactory, tests)
diff --git a/build/android/pylib/monkey/test_options.py b/build/android/pylib/monkey/test_options.py
deleted file mode 100644
index 54d3d084..0000000
--- a/build/android/pylib/monkey/test_options.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Copyright 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Defines the MonkeyOptions named tuple."""
-
-import collections
-
-MonkeyOptions = collections.namedtuple('MonkeyOptions', [
- 'verbose_count',
- 'package',
- 'event_count',
- 'category',
- 'throttle',
- 'seed',
- 'extra_args'])
diff --git a/build/android/pylib/monkey/test_runner.py b/build/android/pylib/monkey/test_runner.py
deleted file mode 100644
index ff4c9400..0000000
--- a/build/android/pylib/monkey/test_runner.py
+++ /dev/null
@@ -1,110 +0,0 @@
-# Copyright 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Runs a monkey test on a single device."""
-
-import logging
-import random
-
-from devil.android import device_errors
-from devil.android.sdk import intent
-from pylib import constants
-from pylib.base import base_test_result
-from pylib.base import base_test_runner
-
-_CHROME_PACKAGE = constants.PACKAGE_INFO['chrome'].package
-
-class TestRunner(base_test_runner.BaseTestRunner):
- """A TestRunner instance runs a monkey test on a single device."""
-
- def __init__(self, test_options, device, _):
- super(TestRunner, self).__init__(device, None)
- self._options = test_options
- self._package = constants.PACKAGE_INFO[self._options.package].package
- self._activity = constants.PACKAGE_INFO[self._options.package].activity
-
- def _LaunchMonkeyTest(self):
- """Runs monkey test for a given package.
-
- Returns:
- Output from the monkey command on the device.
- """
-
- timeout_ms = self._options.event_count * self._options.throttle * 1.5
-
- cmd = ['monkey',
- '-p %s' % self._package,
- ' '.join(['-c %s' % c for c in self._options.category]),
- '--throttle %d' % self._options.throttle,
- '-s %d' % (self._options.seed or random.randint(1, 100)),
- '-v ' * self._options.verbose_count,
- '--monitor-native-crashes',
- '--kill-process-after-error',
- self._options.extra_args,
- '%d' % self._options.event_count]
- return self.device.RunShellCommand(' '.join(cmd), timeout=timeout_ms)
-
- def RunTest(self, test_name):
- """Run a Monkey test on the device.
-
- Args:
- test_name: String to use for logging the test result.
-
- Returns:
- A tuple of (TestRunResults, retry).
- """
- self.device.StartActivity(
- intent.Intent(package=self._package, activity=self._activity,
- action='android.intent.action.MAIN'),
- blocking=True, force_stop=True)
-
- # Chrome crashes are not always caught by Monkey test runner.
- # Verify Chrome has the same PID before and after the test.
- before_pids = self.device.GetPids(self._package)
-
- # Run the test.
- output = ''
- if before_pids:
- if len(before_pids.get(self._package, [])) > 1:
- raise Exception(
- 'At most one instance of process %s expected but found pids: '
- '%s' % (self._package, before_pids))
- output = '\n'.join(self._LaunchMonkeyTest())
- after_pids = self.device.GetPids(self._package)
-
- crashed = True
- if not self._package in before_pids:
- logging.error('Failed to start the process.')
- elif not self._package in after_pids:
- logging.error('Process %s has died.', before_pids[self._package])
- elif before_pids[self._package] != after_pids[self._package]:
- logging.error('Detected process restart %s -> %s',
- before_pids[self._package], after_pids[self._package])
- else:
- crashed = False
-
- results = base_test_result.TestRunResults()
- success_pattern = 'Events injected: %d' % self._options.event_count
- if success_pattern in output and not crashed:
- result = base_test_result.BaseTestResult(
- test_name, base_test_result.ResultType.PASS, log=output)
- else:
- result = base_test_result.BaseTestResult(
- test_name, base_test_result.ResultType.FAIL, log=output)
- if 'chrome' in self._options.package:
- logging.warning('Starting MinidumpUploadService...')
- # TODO(jbudorick): Update this after upstreaming.
- minidump_intent = intent.Intent(
- action='%s.crash.ACTION_FIND_ALL' % _CHROME_PACKAGE,
- package=self._package,
- activity='%s.crash.MinidumpUploadService' % _CHROME_PACKAGE)
- try:
- self.device.RunShellCommand(
- ['am', 'startservice'] + minidump_intent.am_args,
- as_root=True, check_return=True)
- except device_errors.CommandFailedError:
- logging.exception('Failed to start MinidumpUploadService')
-
- results.AddResult(result)
- return results, False
diff --git a/build/android/test_runner.py b/build/android/test_runner.py
index fe9bdbb9..d44d9e3 100755
--- a/build/android/test_runner.py
+++ b/build/android/test_runner.py
@@ -36,8 +36,6 @@
from pylib.linker import setup as linker_setup
from pylib.junit import setup as junit_setup
from pylib.junit import test_dispatcher as junit_dispatcher
-from pylib.monkey import setup as monkey_setup
-from pylib.monkey import test_options as monkey_test_options
from pylib.results import json_results
from pylib.results import report_results
@@ -426,53 +424,32 @@
group = parser.add_argument_group('Monkey Test Options')
group.add_argument(
- '--package', required=True, choices=constants.PACKAGE_INFO.keys(),
- metavar='PACKAGE', help='Package under test.')
+ '--browser', required=True, choices=constants.PACKAGE_INFO.keys(),
+ metavar='BROWSER', help='Browser under test.')
group.add_argument(
'--event-count', default=10000, type=int,
help='Number of events to generate (default: %(default)s).')
group.add_argument(
- '--category', default='',
- help='A list of allowed categories.')
+ '--category', nargs='*', dest='categories', default=[],
+ help='A list of allowed categories. Monkey will only visit activities '
+ 'that are listed with one of the specified categories.')
group.add_argument(
'--throttle', default=100, type=int,
help='Delay between events (ms) (default: %(default)s). ')
group.add_argument(
'--seed', type=int,
- help=('Seed value for pseudo-random generator. Same seed value generates '
- 'the same sequence of events. Seed is randomized by default.'))
+ help='Seed value for pseudo-random generator. Same seed value generates '
+ 'the same sequence of events. Seed is randomized by default.')
group.add_argument(
- '--extra-args', default='',
- help=('String of other args to pass to the command verbatim.'))
-
+ '--repeat', dest='repeat', type=int, default=0,
+ help='Number of times to repeat the specified set of tests.')
+ group.add_argument(
+ '--break-on-failure', '--break_on_failure',
+ dest='break_on_failure', action='store_true',
+ help='Whether to break on failure.')
AddCommonOptions(parser)
AddDeviceOptions(parser)
-def ProcessMonkeyTestOptions(args):
- """Processes all monkey test options.
-
- Args:
- args: argparse.Namespace object.
-
- Returns:
- A MonkeyOptions named tuple which contains all options relevant to
- monkey tests.
- """
- # TODO(jbudorick): Handle this directly in argparse with nargs='+'
- category = args.category
- if category:
- category = args.category.split(',')
-
- # TODO(jbudorick): Get rid of MonkeyOptions.
- return monkey_test_options.MonkeyOptions(
- args.verbose_count,
- args.package,
- args.event_count,
- category,
- args.throttle,
- args.seed,
- args.extra_args)
-
def AddPerfTestOptions(parser):
"""Adds perf test options to |parser|."""
@@ -608,27 +585,6 @@
return exit_code
-def _RunMonkeyTests(args, devices):
- """Subcommand of RunTestsCommands which runs monkey tests."""
- monkey_options = ProcessMonkeyTestOptions(args)
-
- runner_factory, tests = monkey_setup.Setup(monkey_options)
-
- results, exit_code = test_dispatcher.RunTests(
- tests, runner_factory, devices, shard=False, test_timeout=None,
- num_retries=args.num_retries)
-
- report_results.LogFull(
- results=results,
- test_type='Monkey',
- test_package='Monkey')
-
- if args.json_results_file:
- json_results.GenerateJsonResultsFile([results], args.json_results_file)
-
- return exit_code
-
-
def _RunPythonTests(args):
"""Subcommand of RunTestsCommand which runs python unit tests."""
suite_vars = constants.PYTHON_UNIT_TEST_SUITES[args.suite_name]
@@ -678,7 +634,7 @@
return sorted(attached_devices)
-_DEFAULT_PLATFORM_MODE_TESTS = ['gtest', 'instrumentation', 'perf']
+_DEFAULT_PLATFORM_MODE_TESTS = ['gtest', 'instrumentation', 'monkey', 'perf']
def RunTestsCommand(args): # pylint: disable=too-many-return-statements
@@ -718,8 +674,6 @@
return _RunLinkerTests(args, get_devices())
elif command == 'junit':
return _RunJUnitTests(args)
- elif command == 'monkey':
- return _RunMonkeyTests(args, get_devices())
elif command == 'python':
return _RunPythonTests(args)
else:
@@ -731,6 +685,7 @@
'gtest',
'instrumentation',
'junit',
+ 'monkey',
'perf',
]
diff --git a/build/android/test_runner.pydeps b/build/android/test_runner.pydeps
index 8bc5786..a9906bee 100644
--- a/build/android/test_runner.pydeps
+++ b/build/android/test_runner.pydeps
@@ -106,6 +106,7 @@
pylib/local/device/local_device_environment.py
pylib/local/device/local_device_gtest_run.py
pylib/local/device/local_device_instrumentation_test_run.py
+pylib/local/device/local_device_monkey_test_run.py
pylib/local/device/local_device_perf_test_run.py
pylib/local/device/local_device_test_run.py
pylib/local/local_test_server_spawner.py
@@ -113,9 +114,7 @@
pylib/local/machine/local_machine_environment.py
pylib/local/machine/local_machine_junit_test_run.py
pylib/monkey/__init__.py
-pylib/monkey/setup.py
-pylib/monkey/test_options.py
-pylib/monkey/test_runner.py
+pylib/monkey/monkey_test_instance.py
pylib/perf/__init__.py
pylib/perf/perf_test_instance.py
pylib/results/__init__.py
