Replace end-to-end tests with deterministic unit tests

The end-to-end tests could not reliably fail without the fix: a dynamic
test can only be released when the source fails, after which it races
with its parent. The unit tests release the dynamic test only once its
node either awaits it or calls after(), and verify that a test factory
awaits its dynamic tests when its stream fails.

Issue: #6141
Signed-off-by: IWillBurn <eugene.boyarnikov@gmail.com>

jupiter-tests/src/test/java/org/junit/jupiter/engine/descriptor/TestFactoryTestDescriptorTests.java

@@ -12,6 +12,8 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -31,6 +33,7 @@
 import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.engine.config.JupiterConfiguration;
 import org.junit.jupiter.engine.execution.JupiterEngineExecutionContext;
+import org.junit.platform.engine.TestDescriptor;
 import org.junit.platform.engine.TestSource;
 import org.junit.platform.engine.UniqueId;
 import org.junit.platform.engine.support.descriptor.ClasspathResourceSource;
@@ -39,6 +42,7 @@
 import org.junit.platform.engine.support.descriptor.FileSource;
 import org.junit.platform.engine.support.descriptor.MethodSource;
 import org.junit.platform.engine.support.descriptor.UriSource;
+import org.junit.platform.engine.support.hierarchical.Node.DynamicTestExecutor;
 import org.junit.platform.engine.support.hierarchical.OpenTest4JAwareThrowableCollector;
 
 /**
@@ -196,6 +200,27 @@ void streamsFromTestFactoriesShouldBeClosedWhenTheyThrow() {
 			assertTrue(isClosed);
 		}
 
+		@Test
+		void dynamicTestsFromTestFactoriesShouldBeAwaitedWhenTheStreamThrows() throws Exception {
+			var exception = new IllegalStateException("cannot create b");
+			Stream<DynamicTest> dynamicTestStream = Stream.of("a", "b").map(name -> {
+				if (name.equals("b")) {
+					throw exception;
+				}
+				return DynamicTest.dynamicTest(name, () -> {
+				});
+			});
+			prepareMockForTestInstanceWithCustomStream(dynamicTestStream);
+			DynamicTestExecutor dynamicTestExecutor = mock();
+
+			descriptor.invokeTestMethod(context, dynamicTestExecutor);
+
+			var inOrder = inOrder(dynamicTestExecutor);
+			inOrder.verify(dynamicTestExecutor).execute(any(TestDescriptor.class));
+			inOrder.verify(dynamicTestExecutor).awaitFinished();
+			assertThat(context.getThrowableCollector().getThrowable()).isSameAs(exception);
+		}
+
 		private void prepareMockForTestInstanceWithCustomStream(Stream<?> stream) {
 			Stream<?> mockStream = stream.onClose(() -> isClosed = true);
 			when(extensionContext.getRequiredTestInstance()).thenReturn(new CustomStreamTestCase(mockStream));

platform-tests/src/test/java/org/junit/platform/engine/support/hierarchical/HierarchicalTestExecutorTests.java

@@ -20,6 +20,7 @@
 import static org.junit.platform.engine.TestExecutionResult.successful;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.mock;
@@ -29,13 +30,19 @@
 import static org.mockito.Mockito.when;
 import static org.mockito.quality.Strictness.LENIENT;
 
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import org.jspecify.annotations.NonNull;
 import org.jspecify.annotations.NullMarked;
+import org.jspecify.annotations.Nullable;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -616,15 +623,20 @@ void awaitsSubmittedDynamicChildrenBeforeAfterWhenExecuteThrows(ParallelExecutor
 		var dynamicTestDescriptor = spy(new MyLeaf(leafUniqueId.append("dynamic", "child")));
 		root.addChild(child);
 
+		// The dynamic test finishes only once the child either awaits it or calls
+		// after() without awaiting it.
+		var awaitedOrAbandoned = new CountDownLatch(1);
 		var exceptionInExecute = new RuntimeException("execute");
-		var executeThrows = new CountDownLatch(1);
 		when(child.execute(any(), any())).thenAnswer(useDynamicTestExecutor(executor -> {
 			executor.execute(dynamicTestDescriptor);
-			executeThrows.countDown();
 			throw exceptionInExecute;
 		}));
+		doAnswer(invocation -> {
+			awaitedOrAbandoned.countDown();
+			return null;
+		}).when(child).after(any());
 		when(dynamicTestDescriptor.execute(any(), any())).thenAnswer(invocation -> {
-			executeThrows.await(); // finish only after execute() has thrown
+			awaitedOrAbandoned.await();
 			return invocation.getArgument(0);
 		});
 
@@ -634,7 +646,8 @@ void awaitsSubmittedDynamicChildrenBeforeAfterWhenExecuteThrows(ParallelExecutor
 			DefaultParallelExecutionConfigurationStrategy.CONFIG_FIXED_PARALLELISM_PROPERTY_NAME, 2));
 
 		try (var executorService = ParallelHierarchicalTestExecutorServiceFactory.create(parameters)) {
-			createExecutor(executorService).execute().get();
+			createExecutor(countDownOnAwait(executorService, dynamicTestDescriptor, awaitedOrAbandoned)) //
+					.execute().get();
 		}
 
 		var inOrder = inOrder(listener, child);
@@ -647,6 +660,65 @@ void awaitsSubmittedDynamicChildrenBeforeAfterWhenExecuteThrows(ParallelExecutor
 		assertThat(childExecutionResult.getValue().getThrowable().orElseThrow()).isSameAs(exceptionInExecute);
 	}
 
+	/**
+	 * Wrap the supplied executor service so that the supplied latch is counted
+	 * down when the future of the supplied test descriptor's task is awaited.
+	 */
+	private static HierarchicalTestExecutorService countDownOnAwait(HierarchicalTestExecutorService delegate,
+			TestDescriptor testDescriptor, CountDownLatch latch) {
+
+		return new HierarchicalTestExecutorService() {
+
+			@Override
+			public Future<@Nullable Void> submit(TestTask testTask) {
+				var future = delegate.submit(testTask);
+				if (!testDescriptor.equals(testTask.getTestDescriptor())) {
+					return future;
+				}
+				return new Future<>() {
+
+					@Override
+					public boolean cancel(boolean mayInterruptIfRunning) {
+						return future.cancel(mayInterruptIfRunning);
+					}
+
+					@Override
+					public boolean isCancelled() {
+						return future.isCancelled();
+					}
+
+					@Override
+					public boolean isDone() {
+						return future.isDone();
+					}
+
+					@Override
+					public @Nullable Void get() throws InterruptedException, ExecutionException {
+						latch.countDown();
+						return future.get();
+					}
+
+					@Override
+					public @Nullable Void get(long timeout, TimeUnit unit)
+							throws InterruptedException, ExecutionException, TimeoutException {
+						latch.countDown();
+						return future.get(timeout, unit);
+					}
+				};
+			}
+
+			@Override
+			public void invokeAll(List<? extends TestTask> testTasks) {
+				delegate.invokeAll(testTasks);
+			}
+
+			@Override
+			public void close() {
+				delegate.close();
+			}
+		};
+	}
+
 	private Answer<Object> execute(TestDescriptor dynamicChild) {
 		return useDynamicTestExecutor(executor -> executor.execute(dynamicChild));
 	}

platform-tests/src/test/java/org/junit/platform/engine/support/hierarchical/ParallelExecutionIntegrationTests.java

@@ -38,15 +38,13 @@
 import static org.junit.platform.testkit.engine.EventConditions.test;
 import static org.junit.platform.testkit.engine.EventConditions.type;
 import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED;
-import static org.junit.platform.testkit.engine.TestExecutionResultConditions.message;
 
 import java.net.URL;
 import java.net.URLClassLoader;
 import java.time.Instant;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -58,7 +56,6 @@
 import org.assertj.core.api.Condition;
 import org.jspecify.annotations.NullMarked;
 import org.jspecify.annotations.Nullable;
-import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.DynamicContainer;
@@ -78,7 +75,6 @@
 import org.junit.jupiter.api.parallel.Execution;
 import org.junit.jupiter.api.parallel.Isolated;
 import org.junit.jupiter.api.parallel.ResourceLock;
-import org.junit.jupiter.params.Parameter;
 import org.junit.jupiter.params.ParameterizedClass;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.EnumSource;
@@ -190,61 +186,6 @@ void afterHooksAreCalledAfterConcurrentDynamicTestsAreFinished() {
 		assertThat(timestampedEvents.get("afterEach")).isAfterOrEqualTo(timestampedEvents.get("dynamicTestFinished"));
 	}
 
-	@Test
-	void testTemplateAwaitsSubmittedInvocationsWhenItsInvocationStreamFails() {
-		FailingInvocationStreamTestCase.reset();
-
-		var results = executeWithFixedParallelism(3, Map.of(DEFAULT_EXECUTION_MODE_PROPERTY_NAME, "concurrent"),
-			FailingInvocationStreamTestCase.class);
-
-		results.testEvents().assertStatistics(stats -> stats.started(1).succeeded(1));
-		var template = event(container("test-template:"), finishedWithFailure(message("cannot provide b")));
-		assertFinishedBefore(results, event(test(), finishedSuccessfully()), template);
-		assertThat(FailingInvocationStreamTestCase.log).containsExactly("test a finished", "afterAll");
-	}
-
-	@Test
-	void classTemplateAwaitsSubmittedInvocationsWhenItsInvocationStreamFails() {
-		FailingClassInvocationStreamTestCase.reset();
-
-		var results = executeWithFixedParallelism(3, Map.of(DEFAULT_EXECUTION_MODE_PROPERTY_NAME, "concurrent"),
-			FailingClassInvocationStreamTestCase.class);
-
-		results.testEvents().assertStatistics(stats -> stats.started(1).succeeded(1));
-		var invocation = event(container("class-template-invocation:#1"), finishedSuccessfully());
-		var classTemplate = event(container("class-template:"), finishedWithFailure(message("cannot provide b")));
-		assertFinishedBefore(results, event(test(), finishedSuccessfully()), invocation);
-		assertFinishedBefore(results, invocation, classTemplate);
-		assertThat(FailingClassInvocationStreamTestCase.log).containsExactly("test a finished");
-	}
-
-	@Test
-	void afterHooksAreCalledAfterConcurrentDynamicTestsAreFinishedWhenTheTestFactoryStreamFails() {
-		FailingDynamicTestStreamTestCase.reset();
-
-		var results = executeWithFixedParallelism(3, Map.of(DEFAULT_EXECUTION_MODE_PROPERTY_NAME, "concurrent"),
-			List.of(selectMethod(FailingDynamicTestStreamTestCase.class, "testFactory")));
-
-		results.testEvents().assertStatistics(stats -> stats.started(1).succeeded(1));
-		var testFactory = event(container("test-factory:"), finishedWithFailure(message("cannot create b")));
-		assertFinishedBefore(results, event(test(), finishedSuccessfully()), testFactory);
-		assertThat(FailingDynamicTestStreamTestCase.log).containsExactly("dynamic test a finished", "afterEach");
-	}
-
-	@Test
-	void dynamicContainerAwaitsSubmittedChildrenWhenItsChildStreamFails() {
-		FailingDynamicTestStreamTestCase.reset();
-
-		var results = executeWithFixedParallelism(3, Map.of(DEFAULT_EXECUTION_MODE_PROPERTY_NAME, "concurrent"),
-			List.of(selectMethod(FailingDynamicTestStreamTestCase.class, "dynamicContainer")));
-
-		results.testEvents().assertStatistics(stats -> stats.started(1).succeeded(1));
-		var dynamicContainer = event(container("dynamic-container:#1"),
-			finishedWithFailure(message("cannot create b")));
-		assertFinishedBefore(results, event(test(), finishedSuccessfully()), dynamicContainer);
-		assertThat(FailingDynamicTestStreamTestCase.log).containsExactly("dynamic test a finished", "afterEach");
-	}
-
 	/**
 	 * @since 1.4
 	 * @see <a href="https://github.com/junit-team/junit-framework/issues/1688">gh-1688</a>
@@ -613,16 +554,6 @@ private TestDescriptor findFirstTestDescriptor(EngineExecutionResults results, C
 		return results.allEvents().filter(condition::matches).map(Event::getTestDescriptor).findFirst().orElseThrow();
 	}
 
-	private static void assertFinishedBefore(EngineExecutionResults results, Condition<Event> first,
-			Condition<Event> second) {
-		var events = results.allEvents().list();
-		var firstIndex = IntStream.range(0, events.size()).filter(i -> first.matches(events.get(i))).findFirst();
-		var secondIndex = IntStream.range(0, events.size()).filter(i -> second.matches(events.get(i))).findFirst();
-		assertThat(firstIndex).as("event matching %s", first).isPresent();
-		assertThat(secondIndex).as("event matching %s", second).isPresent();
-		assertThat(firstIndex.getAsInt()).as("index of event matching %s", first).isLessThan(secondIndex.getAsInt());
-	}
-
 	private List<Instant> getTimestampsFor(List<Event> events, Condition<Event> condition) {
 		// @formatter:off
 		return events.stream()
@@ -948,100 +879,6 @@ DynamicTest testFactory() {
 		}
 	}
 
-	static class FailingInvocationStreamTestCase {
-		static final List<String> log = new CopyOnWriteArrayList<>();
-		static CountDownLatch argumentsFailed = new CountDownLatch(1);
-
-		static void reset() {
-			log.clear();
-			argumentsFailed = new CountDownLatch(1);
-		}
-
-		@AfterAll
-		static void afterAll() {
-			log.add("afterAll");
-		}
-
-		@ParameterizedTest
-		@org.junit.jupiter.params.provider.MethodSource("arguments")
-		void test(String argument) throws InterruptedException {
-			argumentsFailed.await(); // finish only after the arguments source has thrown
-			log.add("test " + argument + " finished");
-		}
-
-		static Stream<String> arguments() {
-			return Stream.of("a", "b").map(argument -> {
-				if (argument.equals("b")) {
-					argumentsFailed.countDown();
-					throw new IllegalStateException("cannot provide b");
-				}
-				return argument;
-			});
-		}
-	}
-
-	@ParameterizedClass
-	@org.junit.jupiter.params.provider.MethodSource("arguments")
-	static class FailingClassInvocationStreamTestCase {
-		static final List<String> log = new CopyOnWriteArrayList<>();
-
-		static void reset() {
-			log.clear();
-			FailingInvocationStreamTestCase.reset();
-		}
-
-		@Parameter
-		String argument;
-
-		@Test
-		void test() throws InterruptedException {
-			FailingInvocationStreamTestCase.argumentsFailed.await();
-			log.add("test " + argument + " finished");
-		}
-
-		static Stream<String> arguments() {
-			return FailingInvocationStreamTestCase.arguments();
-		}
-	}
-
-	static class FailingDynamicTestStreamTestCase {
-		static final List<String> log = new CopyOnWriteArrayList<>();
-		static CountDownLatch streamFailed = new CountDownLatch(1);
-
-		static void reset() {
-			log.clear();
-			streamFailed = new CountDownLatch(1);
-		}
-
-		@AfterEach
-		void afterEach() {
-			log.add("afterEach");
-		}
-
-		@TestFactory
-		Stream<DynamicTest> testFactory() {
-			return dynamicTests();
-		}
-
-		@TestFactory
-		DynamicContainer dynamicContainer() {
-			return DynamicContainer.dynamicContainer("container", dynamicTests());
-		}
-
-		private static Stream<DynamicTest> dynamicTests() {
-			return Stream.of("a", "b").map(name -> {
-				if (name.equals("b")) {
-					streamFailed.countDown();
-					throw new IllegalStateException("cannot create b");
-				}
-				return dynamicTest(name, () -> {
-					streamFailed.await(); // finish only after the stream has thrown
-					log.add("dynamic test " + name + " finished");
-				});
-			});
-		}
-	}
-
 	@TestMethodOrder(MethodName.class)
 	static class InterruptedThreadTestCase {