Merge pull request #217 from facile-it/detect-phpunit-errors

Detect and report `PHPUnitError` events

CHANGELOG.md

@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## Unreleased
  * ...
 
+## [2.2.1] - 2023-08-30
+ * Add output when failing due to PHPUnit runner errors (i.e. with an empty data provider) [#217](https://github.com/facile-it/paraunit/pull/217)
+
 ## [2.2.0] - 2023-06-08
  * Add `--cobertura` coverage report format, useful for [GitLab test code coverage visualization](https://docs.gitlab.com/ee/ci/testing/test_coverage_visualization.html#php-example) [#206](https://github.com/facile-it/paraunit/pull/206)
 

src/Configuration/ParaunitExtension.php

@@ -12,6 +12,7 @@
 use Paraunit\Logs\TestHook\Passed;
 use Paraunit\Logs\TestHook\PhpDeprecation;
 use Paraunit\Logs\TestHook\PhpUnitDeprecation;
+use Paraunit\Logs\TestHook\PhpUnitError;
 use Paraunit\Logs\TestHook\PhpUnitWarning;
 use Paraunit\Logs\TestHook\PhpWarning;
 use Paraunit\Logs\TestHook\Risky;
@@ -47,6 +48,7 @@ public function bootstrap(Configuration $configuration, Facade $facade, Paramete
  new Skipped(),
  new Passed(),
  new PhpWarning(),
+ new PhpUnitError(),
  new PhpUnitWarning(),
  new TestWarning(),
  new TestRunnerWarning(),

src/Logs/TestHook/PhpUnitError.php

@@ -0,0 +1,21 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Paraunit\Logs\TestHook;
+
+use Paraunit\Logs\ValueObject\LogStatus;
+use Paraunit\Logs\ValueObject\Test;
+use PHPUnit\Event\Test\PhpunitErrorTriggered;
+use PHPUnit\Event\Test\PhpunitErrorTriggeredSubscriber;
+
+/**
+ * @template-extends AbstractTestHook<PhpunitErrorTriggered>
+ */
+class PhpUnitError extends AbstractTestHook implements PhpunitErrorTriggeredSubscriber
+{
+ public function notify(PhpunitErrorTriggered $event): void
+ {
+ $this->write(LogStatus::ErrorTriggered, Test::fromPHPUnitTest($event->test()), $event->message());
+ }
+}

src/Logs/ValueObject/LogStatus.php

@@ -22,6 +22,7 @@ enum LogStatus: string
  case LogTerminated = 'ParaunitLogTerminated';
  case Unknown = 'Unknown';
  case Deprecation = 'Deprecation';
+ case ErrorTriggered = 'ErrorTriggered';
 
  public function toTestStatus(): TestOutcome|TestIssue
  {
@@ -30,6 +31,7 @@ public function toTestStatus(): TestOutcome|TestIssue
  self::Finished,
  self::Started,
  self::LogTerminated => throw new \InvalidArgumentException('Unexpected log status as outcome: ' . $this->value),
+ self::ErrorTriggered,
  self::Errored => TestOutcome::Error,
  self::Failed => TestOutcome::Failure,
  self::MarkedIncomplete => TestOutcome::Incomplete,

tests/Functional/Command/ParallelCommandTest.php

@@ -107,7 +107,7 @@ public function testExecution(): void
  $this->assertStringContainsString(MySQLDeadLockTestStub::class, $output);
  $this->assertStringContainsString(PostgreSQLDeadLockTestStub::class, $output);
  $this->assertNotEquals(0, $exitCode);
- $this->assertStringContainsString('Executed: 16 test classes (21 retried), 25 tests', $output);
+ $this->assertStringContainsString('Executed: 17 test classes (21 retried), 26 tests', $output);
  }
 
  public function testExecutionWithWarning(): void
@@ -190,11 +190,11 @@ public function testExecutionWithDebugEnabled(): void
  $output = $commandTester->getDisplay();
  $this->assertNotEquals(0, $exitCode);
 
- $classExecuted = 16;
+ $classExecuted = 17;
  $processRetried = 21;
  $processesCount = $classExecuted + $processRetried;
  $this->assertStringContainsString(
- sprintf('Executed: %d test classes (%d retried), 25 tests', $classExecuted, $processRetried),
+ sprintf('Executed: %d test classes (%d retried), 26 tests', $classExecuted, $processRetried),
  $output,
  'Precondition failed'
  );
@@ -279,4 +279,23 @@ public function testExecutionWithRandomOrder(): void
  $this->assertEquals(0, $exitCode);
  $this->assertStringContainsString('Executed: 1 test classes, 3 tests', $output);
  }
+
+ public function testRegressionWithPHPUnitError(): void
+ {
+ $application = new Application();
+ $application->add(new ParallelCommand(new ParallelConfiguration()));
+
+ $command = $application->find('run');
+ $commandTester = new CommandTester($command);
+ $exitCode = $commandTester->execute([
+ 'command' => $command->getName(),
+ '--configuration' => $this->getConfigForStubs(),
+ 'stringFilter' => 'PHPUnitError',
+ ]);
+
+ $output = $commandTester->getDisplay();
+ $this->assertNotEquals(0, $exitCode, 'Expecting test failure, got exit code 0');
+ $this->assertStringContainsString('Executed: 1 test classes, 1 tests', $output);
+ $this->assertStringContainsStringIgnoringCase('1 files with ERRORS', $output);
+ }
 }

tests/Functional/Runner/ChunkFileTest.php

@@ -42,11 +42,11 @@ public function testChunkedAllStubsSuite(): void
  'Warnings output:',
  'Deprecations output:',
  '3 chunks with ABNORMAL TERMINATIONS (FATAL ERRORS, SEGFAULTS):',
- '6 chunks with ERRORS:',
- '5 chunks with FAILURES:',
- '5 chunks with WARNINGS:',
+ '7 chunks with ERRORS:',
+ '4 chunks with FAILURES:',
+ '2 chunks with WARNINGS:',
  '1 chunks with DEPRECATIONS:',
- '4 chunks with RETRIED:',
+ '5 chunks with RETRIED:',
  ]);
 
  $this->assertStringContainsString('Tests\Stub\EntityManagerClosedTestStub::testBrokenTest', $outputText);
@@ -58,9 +58,6 @@ public function testChunkedAllStubsSuite(): void
  $this->assertStringContainsString('Tests\Stub\MySQLLockTimeoutTestStub::testBrokenTest', $outputText);
  $this->assertStringContainsString('SQLSTATE[HY000]: General error: 1205 Lock wait timeout exceeded; try restarting transaction', $outputText);
 
- $this->assertStringContainsString('Tests\Stub\PassThenRetryTestStub::testBrokenTest', $outputText);
- $this->assertStringContainsString('SQLSTATE[HY000]: General error: Deadlock found; try restarting transaction', $outputText);
-
  $this->assertStringContainsString('Tests\Stub\PostgreSQLDeadLockTestStub::testBrokenTest', $outputText);
  $this->assertStringContainsString('SQLSTATE[40P01]: Deadlock detected: 7 ERROR: deadlock detected', $outputText);
 

tests/Stub/PHPUnitErrorTestStub.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Stub;
+
+use PHPUnit\Framework\Attributes\DataProvider;
+use PHPUnit\Framework\TestCase;
+
+class PHPUnitErrorTestStub extends TestCase
+{
+ public function testToAvoidWarnings(): void
+ {
+ $this->assertTrue(true);
+ }
+
+ #[DataProvider('emptyDataProvider')]
+ public function testWhichTriggersPHPUnitError(): void
+ {
+ $this->assertTrue(true);
+ }
+
+ public static function emptyDataProvider(): array
+ {
+ return [];
+ }
+}

tests/Unit/Logs/TestHook/PhpUnitErrorTest.php

@@ -0,0 +1,42 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Unit\Logs\TestHook;
+
+use Paraunit\Logs\TestHook\PhpUnitError;
+use Paraunit\Logs\ValueObject\LogStatus;
+use PHPUnit\Event\Test\PhpunitErrorTriggered;
+
+/**
+ * @template-extends AbstractTestHookTestCase<PhpUnitError, PhpunitErrorTriggered>
+ */
+class PhpUnitErrorTest extends AbstractTestHookTestCase
+{
+ protected function createSubscriber(): PhpUnitError
+ {
+ return new PhpUnitError();
+ }
+
+ protected function getExpectedStatus(): LogStatus
+ {
+ return LogStatus::ErrorTriggered;
+ }
+
+ protected function createEvent(): PhpunitErrorTriggered
+ {
+ return new PhpunitErrorTriggered(
+ $this->createTelemetryInfo(),
+ $this->createPHPUnitTestMethod(),
+ $this->getExpectedMessage(),
+ );
+ }
+
+ /**
+ * @return non-empty-string
+ */
+ protected function getExpectedMessage(): string
+ {
+ return 'test Error message';
+ }
+}