Skip to content

Commit

Permalink
[test] Overhaul mode processing in test runner
Browse files Browse the repository at this point in the history 
This simplifies mode processing as follows:
- Passing the --mode parameter is deprecated.
- The build output is now only searched in the --outdir parameter
that was passed (previously some combinations of mode and outdir
were possible).
- The mode is deduced from the build artifacts based on the gn
arguments "is_debug" and "dcheck_always_on".
- Timeouts and status file entries in release mode with dchecks are
treated like in debug mode.

This change was prepared on the infrastructure side by deprecating
the --mode flag and passing --outdir=out/build:
https://crrev.com/c/2426643

Bug: chromium:1132088, v8:10893
Change-Id: I0f34ebc003b220f07df4ecdbf69ea6c06ac1f66a
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2450016
Reviewed-by: Jakob Kummerow <[email protected]>
Commit-Queue: Michael Achenbach <[email protected]>
Cr-Commit-Position: refs/heads/master@{#70363}
  • Loading branch information
@mi-ac
mi-ac authored and Commit Bot committed Oct 7, 2020
1 parent 89af4ca commit 608b732
Show file tree
Hide file tree
Showing 4 changed files with 51 additions and 145 deletions.
131 changes: 34 additions & 97 deletions tools/testrunner/base_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from __future__ import print_function
from functools import reduce

from collections import OrderedDict
from collections import OrderedDict, namedtuple
import json
import multiprocessing
import optparse
Expand Down Expand Up @@ -116,52 +116,38 @@
]


class ModeConfig(object):
def __init__(self, flags, timeout_scalefactor, status_mode, execution_mode):
self.flags = flags
self.timeout_scalefactor = timeout_scalefactor
self.status_mode = status_mode
self.execution_mode = execution_mode

ModeConfig = namedtuple(
'ModeConfig', 'label flags timeout_scalefactor status_mode execution_mode')

DEBUG_FLAGS = ["--nohard-abort", "--enable-slow-asserts", "--verify-heap"]
RELEASE_FLAGS = ["--nohard-abort"]
MODES = {
"debug": ModeConfig(
flags=DEBUG_FLAGS,
timeout_scalefactor=4,
status_mode="debug",
execution_mode="debug",
),
"optdebug": ModeConfig(

DEBUG_MODE = ModeConfig(
label='debug',
flags=DEBUG_FLAGS,
timeout_scalefactor=4,
status_mode="debug",
execution_mode="debug",
),
"release": ModeConfig(
)

RELEASE_MODE = ModeConfig(
label='release',
flags=RELEASE_FLAGS,
timeout_scalefactor=1,
status_mode="release",
execution_mode="release",
),
# Normal trybot release configuration. There, dchecks are always on which
# implies debug is set. Hence, the status file needs to assume debug-like
# behavior/timeouts.
"tryrelease": ModeConfig(
)

# Normal trybot release configuration. There, dchecks are always on which
# implies debug is set. Hence, the status file needs to assume debug-like
# behavior/timeouts.
TRY_RELEASE_MODE = ModeConfig(
label='release+dchecks',
flags=RELEASE_FLAGS,
timeout_scalefactor=1,
status_mode="debug",
execution_mode="release",
),
# This mode requires v8 to be compiled with dchecks and slow dchecks.
"slowrelease": ModeConfig(
flags=RELEASE_FLAGS + ["--enable-slow-asserts"],
timeout_scalefactor=2,
timeout_scalefactor=4,
status_mode="debug",
execution_mode="release",
),
}
)

PROGRESS_INDICATORS = {
'verbose': progress.VerboseProgressIndicator,
Expand Down Expand Up @@ -268,7 +254,6 @@ def __init__(self, basedir=None):
self.basedir = basedir or BASE_DIR
self.outdir = None
self.build_config = None
self.mode_name = None
self.mode_options = None
self.target_os = None

Expand Down Expand Up @@ -305,7 +290,7 @@ def execute(self, sys_args=None):
tests = self._load_testsuite_generators(args, options)
self._setup_env()
print(">>> Running tests for %s.%s" % (self.build_config.arch,
self.mode_name))
self.mode_options.label))
exit_code = self._do_execute(tests, args, options)
if exit_code == utils.EXIT_CODE_FAILURES and options.json_test_results:
print("Force exit code 0 after failures. Json test results file "
Expand Down Expand Up @@ -339,9 +324,6 @@ def _add_parser_default_options(self, parser):
default="out")
parser.add_option("--arch",
help="The architecture to run tests for")
parser.add_option("-m", "--mode",
help="The test mode in which to run (uppercase for builds"
" in CI): %s" % MODES.keys())
parser.add_option("--shell-dir", help="DEPRECATED! Executables from build "
"directory will be used")
parser.add_option("--test-root", help="Root directory of the test suites",
Expand Down Expand Up @@ -427,9 +409,8 @@ def _add_parser_options(self, parser):
def _parse_args(self, parser, sys_args):
options, args = parser.parse_args(sys_args)

if any(map(lambda v: v and ',' in v,
[options.arch, options.mode])): # pragma: no cover
print('Multiple arch/mode are deprecated')
if options.arch and ',' in options.arch: # pragma: no cover
print('Multiple architectures are deprecated')
raise TestRunnerError()

return options, args
Expand Down Expand Up @@ -465,8 +446,7 @@ def _load_build_config(self, options):
# Returns possible build paths in order:
# gn
# outdir
# outdir/arch.mode
# Each path is provided in two versions: <path> and <path>/mode for bots.
# outdir on bots
def _possible_outdirs(self, options):
def outdirs():
if options.gn:
Expand All @@ -475,30 +455,12 @@ def outdirs():

yield options.outdir

# TODO(machenbach): Temporary fallback to legacy outdir. The
# infrastructure switches to the canonical out/build location. But
# bisection will keep working with out/Release or out/Debug for a
# grace period.
if os.path.basename(options.outdir) == 'build':
base_dir = os.path.dirname(options.outdir)
release_dir = os.path.join(base_dir, 'Release')
debug_dir = os.path.join(base_dir, 'Debug')
if os.path.exists(release_dir) and not os.path.exists(debug_dir):
yield release_dir
if os.path.exists(debug_dir) and not os.path.exists(release_dir):
yield debug_dir

if options.arch and options.mode:
yield os.path.join(options.outdir,
'%s.%s' % (options.arch, options.mode))
if os.path.basename(options.outdir) != 'build':
yield os.path.join(options.outdir, 'build')

for outdir in outdirs():
yield os.path.join(self.basedir, outdir)

# bot option
if options.mode:
yield os.path.join(self.basedir, outdir, options.mode)

def _get_gn_outdir(self):
gn_out_dir = os.path.join(self.basedir, DEFAULT_OUT_GN)
latest_timestamp = -1
Expand All @@ -515,29 +477,12 @@ def _get_gn_outdir(self):
return os.path.join(DEFAULT_OUT_GN, latest_config)

def _process_default_options(self, options):
# We don't use the mode for more path-magic.
# Therefore transform the bot mode here to fix build_config value.
if options.mode:
options.mode = self._bot_to_v8_mode(options.mode)

build_config_mode = 'debug' if self.build_config.is_debug else 'release'
if options.mode:
if options.mode not in MODES: # pragma: no cover
print('%s mode is invalid' % options.mode)
raise TestRunnerError()
if MODES[options.mode].execution_mode != build_config_mode:
print ('execution mode (%s) for %s is inconsistent with build config '
'(%s)' % (
MODES[options.mode].execution_mode,
options.mode,
build_config_mode))
raise TestRunnerError()

self.mode_name = options.mode
if self.build_config.is_debug:
self.mode_options = DEBUG_MODE
elif self.build_config.dcheck_always_on:
self.mode_options = TRY_RELEASE_MODE
else:
self.mode_name = build_config_mode

self.mode_options = MODES[self.mode_name]
self.mode_options = RELEASE_MODE

if options.arch and options.arch != self.build_config.arch:
print('--arch value (%s) inconsistent with build config (%s).' % (
Expand All @@ -558,15 +503,6 @@ def _process_default_options(self, options):
options.command_prefix = shlex.split(options.command_prefix)
options.extra_flags = sum(map(shlex.split, options.extra_flags), [])

def _bot_to_v8_mode(self, config):
"""Convert build configs from bots to configs understood by the v8 runner.
V8 configs are always lower case and without the additional _x64 suffix
for 64 bit builds on windows with ninja.
"""
mode = config[:-4] if config.endswith('_x64') else config
return mode.lower()

def _process_options(self, options):
pass

Expand Down Expand Up @@ -714,9 +650,7 @@ def _get_statusfile_variables(self, options):
"is_clang": self.build_config.is_clang,
"is_full_debug": self.build_config.is_full_debug,
"mips_arch_variant": mips_arch_variant,
"mode": self.mode_options.status_mode
if not self.build_config.dcheck_always_on
else "debug",
"mode": self.mode_options.status_mode,
"msan": self.build_config.msan,
"no_harness": options.no_harness,
"no_i18n": self.build_config.no_i18n,
Expand Down Expand Up @@ -827,6 +761,9 @@ def _get_shard_info(self, options):
def _create_progress_indicators(self, test_count, options):
procs = [PROGRESS_INDICATORS[options.progress]()]
if options.json_test_results:
# TODO(machenbach): Deprecate the execution mode. Previously it was meant
# to differentiate several records in the json output. But there is now
# only one record and the mode information is redundant.
procs.append(progress.JsonTestProgressIndicator(
self.framework_name,
self.build_config.arch,
Expand Down
Loading

0 comments on commit 608b732

Please sign in to comment.