diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphTableNames.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphTableNames.php index 787b5d24665..51f14b392cc 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphTableNames.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/ContentGraphTableNames.php @@ -51,9 +51,4 @@ public function contentStream(): string { return $this->tableNamePrefix . '_contentstream'; } - - public function checkpoint(): string - { - return $this->tableNamePrefix . '_checkpoint'; - } } diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php index 28aef165b3f..c1c93e04dd5 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjection.php @@ -59,10 +59,8 @@ use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPublished; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceRebaseFailed; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; -use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\NodeType\NodeTypeName; -use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\NodeTags; @@ -73,7 +71,6 @@ use Neos\ContentRepository\Core\SharedModel\Node\NodeName; use Neos\ContentRepository\Core\SharedModel\Node\ReferenceName; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; /** @@ -91,8 +88,6 @@ final class DoctrineDbalContentGraphProjection implements ContentGraphProjection public const RELATION_DEFAULT_OFFSET = 128; - private DbalCheckpointStorage $checkpointStorage; - public function __construct( private readonly Connection $dbal, private readonly ProjectionContentGraph $projectionContentGraph, @@ -100,11 +95,6 @@ public function __construct( private readonly DimensionSpacePointsRepository $dimensionSpacePointsRepository, private readonly ContentGraphReadModelInterface $contentGraphReadModel ) { - $this->checkpointStorage = new DbalCheckpointStorage( - $this->dbal, - $this->tableNames->checkpoint(), - self::class - ); } public function setUp(): void @@ -118,18 +108,10 @@ public function setUp(): void throw new \RuntimeException(sprintf('Failed to setup projection %s: %s', self::class, $e->getMessage()), 1716478255, $e); } } - $this->checkpointStorage->setUp(); } public function status(): ProjectionStatus { - $checkpointStorageStatus = $this->checkpointStorage->status(); - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { - return ProjectionStatus::error($checkpointStorageStatus->details); - } - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { - return ProjectionStatus::setupRequired($checkpointStorageStatus->details); - } try { $this->dbal->connect(); } catch (\Throwable $e) { @@ -147,17 +129,9 @@ public function status(): ProjectionStatus return ProjectionStatus::ok(); } - public function reset(): void + public function resetState(): void { $this->truncateDatabaseTables(); - - $this->checkpointStorage->acquireLock(); - $this->checkpointStorage->updateAndReleaseLock(SequenceNumber::none()); - } - - public function getCheckpointStorage(): DbalCheckpointStorage - { - return $this->checkpointStorage; } public function getState(): ContentGraphReadModelInterface @@ -165,41 +139,6 @@ public function getState(): ContentGraphReadModelInterface return $this->contentGraphReadModel; } - public function canHandle(EventInterface $event): bool - { - return in_array($event::class, [ - ContentStreamWasClosed::class, - ContentStreamWasCreated::class, - ContentStreamWasForked::class, - ContentStreamWasRemoved::class, - ContentStreamWasReopened::class, - DimensionShineThroughWasAdded::class, - DimensionSpacePointWasMoved::class, - NodeAggregateNameWasChanged::class, - NodeAggregateTypeWasChanged::class, - NodeAggregateWasMoved::class, - NodeAggregateWasRemoved::class, - NodeAggregateWithNodeWasCreated::class, - NodeGeneralizationVariantWasCreated::class, - NodePeerVariantWasCreated::class, - NodePropertiesWereSet::class, - NodeReferencesWereSet::class, - NodeSpecializationVariantWasCreated::class, - RootNodeAggregateDimensionsWereUpdated::class, - RootNodeAggregateWithNodeWasCreated::class, - RootWorkspaceWasCreated::class, - SubtreeWasTagged::class, - SubtreeWasUntagged::class, - WorkspaceBaseWorkspaceWasChanged::class, - WorkspaceRebaseFailed::class, - WorkspaceWasCreated::class, - WorkspaceWasDiscarded::class, - WorkspaceWasPublished::class, - WorkspaceWasRebased::class, - WorkspaceWasRemoved::class, - ]) || $event instanceof EmbedsContentStreamId; - } - public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void { match ($event::class) { @@ -232,7 +171,7 @@ public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void WorkspaceWasPublished::class => $this->whenWorkspaceWasPublished($event), WorkspaceWasRebased::class => $this->whenWorkspaceWasRebased($event), WorkspaceWasRemoved::class => $this->whenWorkspaceWasRemoved($event), - default => $event instanceof EmbedsContentStreamId || throw new \InvalidArgumentException(sprintf('Unsupported event %s', get_debug_type($event))), + default => null, }; if ( $event instanceof EmbedsContentStreamId diff --git a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php index a5c4d6ae25e..f63308efc43 100644 --- a/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php +++ b/Neos.ContentGraph.DoctrineDbalAdapter/src/DoctrineDbalContentGraphProjectionFactory.php @@ -8,7 +8,7 @@ use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Repository\DimensionSpacePointsRepository; use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Repository\NodeFactory; use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Repository\ProjectionContentGraph; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; /** @@ -24,8 +24,7 @@ public function __construct( } public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, - array $options, + SubscriberFactoryDependencies $projectionFactoryDependencies, ): DoctrineDbalContentGraphProjection { $tableNames = ContentGraphTableNames::create( $projectionFactoryDependencies->contentRepositoryId @@ -35,7 +34,7 @@ public function build( $nodeFactory = new NodeFactory( $projectionFactoryDependencies->contentRepositoryId, - $projectionFactoryDependencies->propertyConverter, + $projectionFactoryDependencies->getPropertyConverter(), $dimensionSpacePointsRepository ); diff --git a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php index 8c6ff9118b8..5961442b9e6 100644 --- a/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php +++ b/Neos.ContentGraph.PostgreSQLAdapter/src/Domain/Projection/HypergraphProjection.php @@ -15,7 +15,6 @@ namespace Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection; use Doctrine\DBAL\Connection; -use Doctrine\DBAL\Schema\AbstractSchemaManager; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\ContentStreamForking; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\NodeCreation; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\NodeModification; @@ -26,7 +25,6 @@ use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\NodeVariation; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\Feature\SubtreeTagging; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\SchemaBuilder\HypergraphSchemaBuilder; -use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\EventStore\EventInterface; use Neos\ContentRepository\Core\Feature\ContentStreamForking\Event\ContentStreamWasForked; use Neos\ContentRepository\Core\Feature\NodeCreation\Event\NodeAggregateWithNodeWasCreated; @@ -41,12 +39,10 @@ use Neos\ContentRepository\Core\Feature\RootNodeCreation\Event\RootNodeAggregateWithNodeWasCreated; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; -use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; -use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; -use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; -use Neos\EventStore\Model\Event\SequenceNumber; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\EventStore\Model\EventEnvelope; /** @@ -66,7 +62,6 @@ final class HypergraphProjection implements ContentGraphProjectionInterface use NodeTypeChange; use NodeVariation; - private DbalCheckpointStorage $checkpointStorage; private ProjectionHypergraph $projectionHypergraph; public function __construct( @@ -75,11 +70,6 @@ public function __construct( private readonly ContentGraphReadModelInterface $contentGraphReadModel ) { $this->projectionHypergraph = new ProjectionHypergraph($this->dbal, $this->tableNamePrefix); - $this->checkpointStorage = new DbalCheckpointStorage( - $this->dbal, - $this->tableNamePrefix . '_checkpoint', - self::class - ); } @@ -97,18 +87,10 @@ public function setUp(): void create index if not exists restriction_affected on ' . $this->tableNamePrefix . '_restrictionhyperrelation using gin (affectednodeaggregateids); '); - $this->checkpointStorage->setUp(); } public function status(): ProjectionStatus { - $checkpointStorageStatus = $this->checkpointStorage->status(); - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { - return ProjectionStatus::error($checkpointStorageStatus->details); - } - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { - return ProjectionStatus::setupRequired($checkpointStorageStatus->details); - } try { $this->getDatabaseConnection()->connect(); } catch (\Throwable $e) { @@ -135,12 +117,9 @@ private function determineRequiredSqlStatements(): array return DbalSchemaDiff::determineRequiredSqlStatements($this->dbal, $schema); } - public function reset(): void + public function resetState(): void { $this->truncateDatabaseTables(); - - $this->checkpointStorage->acquireLock(); - $this->checkpointStorage->updateAndReleaseLock(SequenceNumber::none()); } private function truncateDatabaseTables(): void @@ -151,39 +130,6 @@ private function truncateDatabaseTables(): void $this->dbal->executeQuery('TRUNCATE table ' . $this->tableNamePrefix . '_restrictionhyperrelation'); } - public function canHandle(EventInterface $event): bool - { - return in_array($event::class, [ - // ContentStreamForking - ContentStreamWasForked::class, - // NodeCreation - RootNodeAggregateWithNodeWasCreated::class, - NodeAggregateWithNodeWasCreated::class, - // SubtreeTagging - SubtreeWasTagged::class, - SubtreeWasUntagged::class, - // NodeModification - NodePropertiesWereSet::class, - // NodeReferencing - NodeReferencesWereSet::class, - // NodeRemoval - NodeAggregateWasRemoved::class, - // NodeRenaming - NodeAggregateNameWasChanged::class, - // NodeTypeChange - NodeAggregateTypeWasChanged::class, - // NodeVariation - NodeSpecializationVariantWasCreated::class, - NodeGeneralizationVariantWasCreated::class, - NodePeerVariantWasCreated::class, - // TODO: not yet supported: - //ContentStreamWasRemoved::class, - //DimensionSpacePointWasMoved::class, - //DimensionShineThroughWasAdded::class, - //NodeAggregateWasMoved::class, - ]); - } - public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void { match ($event::class) { @@ -209,7 +155,7 @@ public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void NodeSpecializationVariantWasCreated::class => $this->whenNodeSpecializationVariantWasCreated($event), NodeGeneralizationVariantWasCreated::class => $this->whenNodeGeneralizationVariantWasCreated($event), NodePeerVariantWasCreated::class => $this->whenNodePeerVariantWasCreated($event), - default => throw new \InvalidArgumentException(sprintf('Unsupported event %s', get_debug_type($event))), + default => null, }; } @@ -228,11 +174,6 @@ public function inSimulation(\Closure $fn): mixed } } - public function getCheckpointStorage(): DbalCheckpointStorage - { - return $this->checkpointStorage; - } - public function getState(): ContentGraphReadModelInterface { return $this->contentGraphReadModel; diff --git a/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php b/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php index 3d3c002c094..b2775d8b69c 100644 --- a/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php +++ b/Neos.ContentGraph.PostgreSQLAdapter/src/HypergraphProjectionFactory.php @@ -7,7 +7,7 @@ use Doctrine\DBAL\Connection; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Projection\HypergraphProjection; use Neos\ContentGraph\PostgreSQLAdapter\Domain\Repository\NodeFactory; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; @@ -28,8 +28,7 @@ public static function graphProjectionTableNamePrefix( } public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, - array $options, + SubscriberFactoryDependencies $projectionFactoryDependencies, ): HypergraphProjection { $tableNamePrefix = self::graphProjectionTableNamePrefix( $projectionFactoryDependencies->contentRepositoryId @@ -37,7 +36,7 @@ public function build( $nodeFactory = new NodeFactory( $projectionFactoryDependencies->contentRepositoryId, - $projectionFactoryDependencies->propertyConverter + $projectionFactoryDependencies->getPropertyConverter() ); return new HypergraphProjection( diff --git a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php index 039156ef9da..d491a718fff 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHook.php @@ -17,12 +17,13 @@ use Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\Dto\TraceEntries; use Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester\Dto\TraceEntryType; use Neos\ContentRepository\Core\EventStore\EventInterface; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; use Neos\Flow\Annotations as Flow; /** - * We had some race conditions in projections, where {@see \Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage} was not working properly. + * We had some race conditions in projections * We saw some non-deterministic, random errors when running the tests - unluckily only on Linux, not on OSX: * On OSX, forking a new subprocess in {@see SubprocessProjectionCatchUpTrigger} is *WAY* slower than in Linux; * and thus the race conditions which appears if two projector instances of the same class run concurrently @@ -73,7 +74,7 @@ * * When {@see onBeforeEvent} is called, we know that we are inside applyEvent() in the diagram above, * thus we know the lock *HAS* been acquired. - * When {@see onBeforeBatchCompleted}is called, we know the lock will be released directly afterwards. + * When {@see onAfterCatchUp}is called, we know the lock will be released directly afterwards. * * We track these timings across processes in a single Redis Stream. Because Redis is single-threaded, * we can be sure that we observe the correct, total order of interleavings *across multiple processes* @@ -107,7 +108,7 @@ final class RaceTrackerCatchUpHook implements CatchUpHookInterface protected $configuration; private bool $inCriticalSection = false; - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { RedisInterleavingLogger::connect($this->configuration['redis']['host'], $this->configuration['redis']['port']); } @@ -126,7 +127,11 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event { } - public function onBeforeBatchCompleted(): void + public function onAfterBatchCompleted(): void + { + } + + public function onAfterCatchUp(): void { // we only want to track relevant lock release calls (i.e. if we were in the event processing loop before) if ($this->inCriticalSection) { @@ -134,8 +139,4 @@ public function onBeforeBatchCompleted(): void RedisInterleavingLogger::trace(TraceEntryType::LockWillBeReleasedIfItWasAcquiredBefore); } } - - public function onAfterCatchUp(): void - { - } } diff --git a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHookFactory.php b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHookFactory.php index 389d7a324a5..829ed582f96 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHookFactory.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/ProjectionRaceConditionTester/RaceTrackerCatchUpHookFactory.php @@ -14,9 +14,9 @@ namespace Neos\ContentRepository\BehavioralTests\ProjectionRaceConditionTester; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; /** diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php index 9f8d56cda7f..2fb4fe8ac9b 100644 --- a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/Behavior/CRBehavioralTestsSubjectProvider.php @@ -18,11 +18,14 @@ use Behat\Gherkin\Node\TableNode; use Doctrine\DBAL\Connection; use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Helpers\GherkinTableNodeBasedContentDimensionSource; use Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory; use Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory; use Neos\EventStore\EventStoreInterface; +use PHPUnit\Framework\Assert; use Symfony\Component\Yaml\Yaml; /** @@ -178,17 +181,26 @@ protected function setUpContentRepository(ContentRepositoryId $contentRepository * Catch Up process and the testcase reset. */ $contentRepository = $this->createContentRepository($contentRepositoryId); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); if (!in_array($contentRepository->id, self::$alreadySetUpContentRepositories)) { - $contentRepository->setUp(); + $result = $contentRepositoryMaintainer->setUp(); + Assert::assertNull($result); self::$alreadySetUpContentRepositories[] = $contentRepository->id; } + // todo we TRUNCATE here and do not want to use $contentRepositoryMaintainer->prune(); here as it would not reset the autoincrement sequence number making some assertions impossible /** @var EventStoreInterface $eventStore */ $eventStore = (new \ReflectionClass($contentRepository))->getProperty('eventStore')->getValue($contentRepository); /** @var Connection $databaseConnection */ $databaseConnection = (new \ReflectionClass($eventStore))->getProperty('connection')->getValue($eventStore); $eventTableName = sprintf('cr_%s_events', $contentRepositoryId->value); $databaseConnection->executeStatement('TRUNCATE ' . $eventTableName); - $contentRepository->resetProjectionStates(); + + /** @var SubscriptionEngine $subscriptionEngine */ + $subscriptionEngine = (new \ReflectionClass($contentRepositoryMaintainer))->getProperty('subscriptionEngine')->getValue($contentRepositoryMaintainer); + $result = $subscriptionEngine->reset(); + Assert::assertNull($result->errors); + $result = $subscriptionEngine->boot(); + Assert::assertNull($result->errors); return $contentRepository; } diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php new file mode 100644 index 00000000000..556e2d34d41 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjection.php @@ -0,0 +1,136 @@ + + * @internal + * @Flow\Proxy(false) + */ +final class DebugEventProjection implements ProjectionInterface +{ + private DebugEventProjectionState $state; + + private \Closure|null $saboteur = null; + + /** + * @var array + */ + private array $additionalColumnsForSchema = []; + + public function __construct( + private string $tableNamePrefix, + private Connection $dbal + ) { + $this->state = new DebugEventProjectionState($this->tableNamePrefix, $this->dbal); + } + + public function setUp(): void + { + foreach ($this->determineRequiredSqlStatements() as $statement) { + $this->dbal->executeStatement($statement); + } + if ($this->saboteur) { + ($this->saboteur)(); + } + } + + public function status(): ProjectionStatus + { + $requiredSqlStatements = $this->determineRequiredSqlStatements(); + if ($requiredSqlStatements !== []) { + return ProjectionStatus::setupRequired(sprintf('Requires %d SQL statements', count($requiredSqlStatements))); + } + return ProjectionStatus::ok(); + } + + /** + * @return array + */ + private function determineRequiredSqlStatements(): array + { + $schemaManager = $this->dbal->createSchemaManager(); + + $table = new Table($this->tableNamePrefix, [ + (new Column('sequencenumber', Type::getType(Types::INTEGER))), + (new Column('stream', Type::getType(Types::STRING))), + (new Column('type', Type::getType(Types::STRING))), + ...$this->additionalColumnsForSchema + ]); + + $table->setPrimaryKey([ + 'sequencenumber' + ]); + + $schema = DbalSchemaFactory::createSchemaWithTables($schemaManager, [$table]); + $statements = DbalSchemaDiff::determineRequiredSqlStatements($this->dbal, $schema); + + return $statements; + } + + public function resetState(): void + { + $this->dbal->executeStatement('TRUNCATE ' . $this->tableNamePrefix); + } + + public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void + { + try { + $this->dbal->insert($this->tableNamePrefix, [ + 'sequencenumber' => $eventEnvelope->sequenceNumber->value, + 'stream' => $eventEnvelope->streamName->value, + 'type' => $eventEnvelope->event->type->value, + ]); + } catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $exception) { + throw new \RuntimeException(sprintf('Must not happen! Debug projection detected duplicate event %s of type %s', $eventEnvelope->sequenceNumber->value, $eventEnvelope->event->type->value), 1732360282, $exception); + } + if ($this->saboteur) { + ($this->saboteur)($eventEnvelope); + } + } + + public function getState(): ProjectionStateInterface + { + return $this->state; + } + + public function injectSaboteur(\Closure $saboteur): void + { + $this->saboteur = $saboteur; + } + + public function killSaboteur(): void + { + $this->saboteur = null; + } + + public function schemaNeedsAdditionalColumn(string $name): void + { + $this->additionalColumnsForSchema[$name] = (new Column($name, Type::getType(Types::STRING)))->setNotnull(false); + } + + public function dropTables(): void + { + $this->dbal->executeStatement('DROP TABLE ' . $this->tableNamePrefix); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php new file mode 100644 index 00000000000..2c739009f9e --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Classes/TestSuite/DebugEventProjectionState.php @@ -0,0 +1,44 @@ + + */ + public function findAppliedSequenceNumbers(): array + { + return array_map( + fn (int $value) => SequenceNumber::fromInteger($value), + $this->findAppliedSequenceNumberValues() + ); + } + + /** + * @return array + */ + public function findAppliedSequenceNumberValues(): array + { + return array_map( + fn ($value) => (int)$value['sequenceNumber'], + $this->dbal->fetchAllAssociative("SELECT sequenceNumber from {$this->tableNamePrefix}") + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml index 2d0fe1e74c0..24d25b27065 100644 --- a/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml +++ b/Neos.ContentRepository.BehavioralTests/Configuration/Testing/Settings.yaml @@ -31,10 +31,52 @@ Neos: factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeAuthProviderFactory clock: factoryObjectName: Neos\ContentRepositoryRegistry\Factory\Clock\SystemClockFactory + subscriptionStore: + factoryObjectName: Neos\ContentRepositoryRegistry\Factory\SubscriptionStore\SubscriptionStoreFactory propertyConverters: {} contentGraphProjection: factoryObjectName: Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory + catchUpHooks: {} + projections: + 'Neos.Testing:DebugProjection': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory + options: + instanceId: debug + t_subscription: + eventStore: + factoryObjectName: Neos\ContentRepositoryRegistry\Factory\EventStore\DoctrineEventStoreFactory + nodeTypeManager: + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory + contentDimensionSource: + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory + authProvider: + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeAuthProviderFactory + clock: + factoryObjectName: Neos\ContentRepositoryRegistry\Factory\Clock\SystemClockFactory + subscriptionStore: + factoryObjectName: Neos\ContentRepositoryRegistry\Factory\SubscriptionStore\SubscriptionStoreFactory + propertyConverters: {} + contentGraphProjection: + factoryObjectName: Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory + catchUpHooks: {} + projections: + 'Vendor.Package:FakeProjection': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory + options: + instanceId: default + catchUpHooks: + 'Vendor.Package:FakeCatchupHook': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeCatchUpHookFactory + 'Vendor.Package:SecondFakeProjection': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory + options: + instanceId: second + catchUpHooks: + 'Vendor.Package:SecondFakeCatchupHook': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeCatchUpHookFactory + 'Vendor.Package:AdditionalSecondFakeCatchupHook': + factoryObjectName: Neos\ContentRepository\TestSuite\Fakes\FakeCatchUpHookFactory2 Flow: object: includeClasses: diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php new file mode 100644 index 00000000000..c630985ba7c --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/AbstractSubscriptionEngineTestCase.php @@ -0,0 +1,231 @@ +getObject(Connection::class)->getDatabasePlatform() instanceof PostgreSQLPlatform) { + $this->markTestSkipped('TODO: The content graph is not available in postgres currently: https://github.com/neos/neos-development-collection/issues/3855'); + } + + $this->resetDatabase( + $this->getObject(Connection::class), + self::$contentRepositoryId, + keepSchema: true + ); + + $this->fakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); + $this->fakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); + + FakeProjectionFactory::setProjection( + 'default', + $this->fakeProjection + ); + + if (!isset($this->secondFakeProjection)) { + $this->secondFakeProjection = new DebugEventProjection( + sprintf('cr_%s_debug_projection', self::$contentRepositoryId->value), + $this->getObject(Connection::class) + ); + } + + FakeProjectionFactory::setProjection( + 'second', + $this->secondFakeProjection + ); + + $this->catchupHookForFakeProjection = $this->getMockBuilder(CatchUpHookInterface::class)->getMock(); + FakeCatchUpHookFactory::setCatchupHook( + $this->fakeProjection->getState(), + $this->catchupHookForFakeProjection + ); + + $this->catchupHookForSecondFakeProjection = $this->getMockBuilder(CatchUpHookInterface::class)->getMock(); + FakeCatchUpHookFactory::setCatchupHook( + $this->secondFakeProjection->getState(), + $this->catchupHookForSecondFakeProjection + ); + + $this->additionalCatchupHookForSecondFakeProjection = $this->getMockBuilder(CatchUpHookInterface::class)->getMock(); + FakeCatchUpHookFactory2::setCatchupHook( + $this->secondFakeProjection->getState(), + $this->additionalCatchupHookForSecondFakeProjection + ); + + FakeNodeTypeManagerFactory::setConfiguration([]); + FakeContentDimensionSourceFactory::setWithoutDimensions(); + + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance(self::$contentRepositoryId); + + $this->setupContentRepositoryDependencies(self::$contentRepositoryId); + } + + final protected function setupContentRepositoryDependencies(ContentRepositoryId $contentRepositoryId) + { + $this->contentRepository = $this->getObject(ContentRepositoryRegistry::class)->get( + $contentRepositoryId + ); + + $subscriptionEngineAndEventStoreAccessor = new class implements ContentRepositoryServiceFactoryInterface { + public EventStoreInterface|null $eventStore; + public SubscriptionEngine|null $subscriptionEngine; + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface + { + $this->eventStore = $serviceFactoryDependencies->eventStore; + $this->subscriptionEngine = $serviceFactoryDependencies->subscriptionEngine; + return new class implements ContentRepositoryServiceInterface + { + }; + } + }; + $this->getObject(ContentRepositoryRegistry::class)->buildService($contentRepositoryId, $subscriptionEngineAndEventStoreAccessor); + $this->eventStore = $subscriptionEngineAndEventStoreAccessor->eventStore; + $this->subscriptionEngine = $subscriptionEngineAndEventStoreAccessor->subscriptionEngine; + } + + final protected function resetDatabase(Connection $connection, ContentRepositoryId $contentRepositoryId, bool $keepSchema): void + { + $preDeleteStatement = match (true) { + $connection->getDatabasePlatform() instanceof AbstractMySQLPlatform => 'SET FOREIGN_KEY_CHECKS = 0;', + default => '', + }; + + if ($preDeleteStatement !== '') { + $connection->prepare($preDeleteStatement)->executeStatement(); + } + + $truncateOrDropStatement = match (true) { + $connection->getDatabasePlatform() instanceof PostgreSQLPlatform => '%s TABLE `%s` CASCADE', + default => '%s TABLE `%s`', + }; + + foreach ($connection->createSchemaManager()->listTableNames() as $tableName) { + if (!str_starts_with($tableName, sprintf('cr_%s_', $contentRepositoryId->value))) { + // speedup deletion, only delete current cr + continue; + } + // truncate is faster + $sql = sprintf($truncateOrDropStatement, $keepSchema ? 'TRUNCATE' : 'DROP', $tableName); + $connection->prepare($sql)->executeStatement(); + } + + $postDeleteStatement = match (true) { + $connection->getDatabasePlatform() instanceof AbstractMySQLPlatform => 'SET FOREIGN_KEY_CHECKS = 1;', + default => '', + }; + + if ($postDeleteStatement !== '') { + $connection->prepare($postDeleteStatement)->executeStatement(); + } + } + + final protected function subscriptionStatus(string $subscriptionId): ProjectionSubscriptionStatus|DetachedSubscriptionStatus|null + { + return $this->subscriptionEngine->subscriptionStatus(SubscriptionEngineCriteria::create(ids: [SubscriptionId::fromString($subscriptionId)]))->first(); + } + + final protected function commitExampleContentStreamEvent(): void + { + $this->eventStore->commit( + ContentStreamEventStreamName::fromContentStreamId($cs = ContentStreamId::create())->getEventStreamName(), + new Event( + Event\EventId::create(), + Event\EventType::fromString('ContentStreamWasCreated'), + Event\EventData::fromString(json_encode(['contentStreamId' => $cs->value])) + ), + ExpectedVersion::NO_STREAM() + ); + } + + final protected function expectOkayStatus(string $subscriptionId, SubscriptionStatus $status, SequenceNumber $sequenceNumber): void + { + $actual = $this->subscriptionStatus($subscriptionId); + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString($subscriptionId), + subscriptionStatus: $status, + subscriptionPosition: $sequenceNumber, + subscriptionError: null, + setupStatus: ProjectionStatus::ok(), + ), + $actual + ); + } + + /** + * @template T of object + * @param class-string $className + * + * @return T + */ + final protected function getObject(string $className): object + { + return Bootstrap::$staticObjectManager->get($className); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php new file mode 100644 index 00000000000..8dabe601728 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookErrorTest.php @@ -0,0 +1,601 @@ +eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit two events. we expect that the hook will throw for both events but the catchup is NOT halted + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + + $exception = new \RuntimeException('This catchup hook is kaputt.'); + $this->catchupHookForSecondFakeProjection->expects($invokedCount = self::exactly(2))->method('onBeforeEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($invokedCount, $exception) { + match ($invokedCount->getInvocationCount()) { + 1 => self::assertSame(1, $eventEnvelope->sequenceNumber->value), + 2 => self::assertSame(2, $eventEnvelope->sequenceNumber->value), + }; + throw $exception; + }); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "onBeforeEvent" failed: "": This catchup hook is kaputt.', + 1733243960, + $exception + ); + + // two errors for both of the events + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals( + ProcessedResult::failed( + 2, + Errors::fromArray([ + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + $expectedWrappedException, + SequenceNumber::fromInteger(1) + ), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + null, + SequenceNumber::fromInteger(2) + ), + ]) + ), + $result + ); + + // both events are applied still + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function error_onAfterEvent_isIgnoredAndCollected() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit two events. we expect that the hook will throw for both events but the catchup is NOT halted + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $exception = new \RuntimeException('This catchup hook is kaputt.'); + $this->catchupHookForSecondFakeProjection->expects($invokedCount = self::exactly(2))->method('onAfterEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($invokedCount, $exception) { + match ($invokedCount->getInvocationCount()) { + 1 => self::assertSame(1, $eventEnvelope->sequenceNumber->value), + 2 => self::assertSame(2, $eventEnvelope->sequenceNumber->value), + }; + throw $exception; + }); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); // todo assert no parameters?! + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "onAfterEvent" failed: "": This catchup hook is kaputt.', + 1733243960, + $exception + ); + + // two errors for both of the events + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals( + ProcessedResult::failed( + 2, + Errors::fromArray([ + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + $expectedWrappedException, + SequenceNumber::fromInteger(1) + ), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + null, + SequenceNumber::fromInteger(2) + ), + ]) + ), + $result + ); + + // both events are applied still + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function error_onBeforeCatchUp_isIgnoredAndCollected() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit two events. we expect that the hook will throw for both events but the catchup is NOT halted + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willThrowException( + $exception = new \RuntimeException('This catchup hook is kaputt.') + ); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onBeforeEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "onBeforeCatchUp" failed: "": This catchup hook is kaputt.', + 1733243960, + $exception + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals( + ProcessedResult::failed( + 2, + Errors::fromArray([ + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + $expectedWrappedException, + null + ), + ]) + ), + $result + ); + + // both events are applied still + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function error_onAfterBatchCompleted_isIgnoredAndCollected() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit two events. we expect that the hook will throw for both events but the catchup is NOT halted + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp'); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onBeforeEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted')->willThrowException( + $exception = new \RuntimeException('This catchup hook is kaputt.') + ); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "onAfterBatchCompleted" failed: "": This catchup hook is kaputt.', + 1733243960, + $exception + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals( + ProcessedResult::failed( + 2, + Errors::fromArray([ + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + $expectedWrappedException, + null + ), + ]) + ), + $result + ); + + // both events are applied still + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function error_onAfterCatchUp_isIgnoredAndCollected() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit two events. we expect that the hook will throw for both events but the catchup is NOT halted + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp'); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onBeforeEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::exactly(2))->method('onAfterEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp')->willThrowException( + $exception = new \RuntimeException('This catchup hook is kaputt.') + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "onAfterCatchUp" failed: "": This catchup hook is kaputt.', + 1733243960, + $exception + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals( + ProcessedResult::failed( + 2, + Errors::fromArray([ + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + $expectedWrappedException, + null + ), + ]) + ), + $result + ); + + // both events are applied still + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function error_onAfterCatchUp_isIgnoredAndCollected_withProjectionError() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit two events. we expect that the hook will throw for both events but the catchup is NOT halted + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp'); + // only the onBeforeEvent hook will be invoked as afterward the projection errored + $this->catchupHookForSecondFakeProjection->expects(self::exactly(1))->method('onBeforeEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp')->willThrowException( + $exception = new \RuntimeException('This catchup hook is kaputt.') + ); + + $innerException = new \RuntimeException('Projection is kaputt.'); + $this->secondFakeProjection->injectSaboteur(fn () => throw $innerException); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "onAfterCatchUp" failed: "": This catchup hook is kaputt.', + 1733243960, + $exception + ); + + // two errors for both of the events + $result = $this->subscriptionEngine->catchUpActive(); + + self::assertEquals( + ProcessedResult::failed( + 2, + Errors::fromArray([ + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $innerException->getMessage(), + $innerException, + SequenceNumber::fromInteger(1) + ), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + null, + null + ), + ]) + ), + $result + ); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $innerException), + setupStatus: ProjectionStatus::ok(), + ); + + // projection is marked as error + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + // partially applied event because the error is thrown at the end and the projection is not rolled back + self::assertEquals( + [1], + $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues() + ); + } + + /** @test */ + public function error_onAfterEvent_stopsEngineAfterFirstBatch() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + + // commit two events. we expect that the hook will throw the first event and due to the batching its halted + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $exception = new \RuntimeException('This catchup hook is kaputt.'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->willThrowException( + $exception + ); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(0)); + + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "onAfterEvent" failed: "": This catchup hook is kaputt.', + 1733243960, + $exception + ); + + // one error + $result = $this->subscriptionEngine->boot(batchSize: 1); + self::assertEquals( + ProcessedResult::failed( + 1, + Errors::fromArray([ + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + $expectedWrappedException, + SequenceNumber::fromInteger(1) + ), + ]) + ), + $result + ); + + // only one event is applied + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(1)); + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function error_onAfterEvent_withMultipleFailingHooks() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $firstException = new \RuntimeException('First catchup hook is kaputt.'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->willThrowException( + $firstException + ); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $secondException = new \RuntimeException('Second catchup hook is kaputt.'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->willThrowException( + $secondException + ); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(0)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(0)); + + $result = $this->subscriptionEngine->boot(); + self::assertEquals( + ProcessedResult::failed( + 1, + Errors::fromArray([ + Error::create( + SubscriptionId::fromString('Vendor.Package:FakeProjection'), + 'Hook "onAfterEvent" failed: "": First catchup hook is kaputt.', + new CatchUpHookFailed( + 'Hook "onAfterEvent" failed: "": First catchup hook is kaputt.', + 1733243960, + $firstException + ), + SequenceNumber::fromInteger(1) + ), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + 'Hook "onAfterEvent" failed: "": Second catchup hook is kaputt.', + null, + SequenceNumber::fromInteger(1) + ), + ]) + ), + $result + ); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function error_onAfterEvent_withMultipleFailingHooksOnOneProjection() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $firstException = new \RuntimeException('First catchup hook is kaputt.'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->willThrowException( + $firstException + ); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $secondException = new \RuntimeException('Second catchup hook is kaputt.'); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->willThrowException( + $secondException + ); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(0)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(0)); + + $result = $this->subscriptionEngine->boot(); + self::assertEquals( + ProcessedResult::failed( + 1, + Errors::fromArray([ + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $message = 'Hook "onAfterEvent" failed: "": First catchup hook is kaputt.;' . PHP_EOL . + '"": Second catchup hook is kaputt.', + new CatchUpHookFailed( + $message, + 1733243960, + $firstException + ), + SequenceNumber::fromInteger(1) + ), + ]) + ), + $result + ); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php new file mode 100644 index 00000000000..6693f1c2563 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookTest.php @@ -0,0 +1,304 @@ +eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $expectNoHandledEvents = fn () => self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectOneHandledEvent = fn () => self::assertEquals( + [ + SequenceNumber::fromInteger(1) + ], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $expectNoTransactionActive = fn () => self::assertFalse( + $this->getObject(Connection::class)->isTransactionActive(), 'Expected no transaction to be active' + ); + + $expectTransactionActive = fn () => self::assertTrue( + $this->getObject(Connection::class)->isTransactionActive(), 'Expected transaction to be active' + ); + + // first projection hooks + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willReturnCallback($expectTransactionActive); + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectTransactionActive); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectTransactionActive); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted')->willReturnCallback($expectNoTransactionActive); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp')->willReturnCallback($expectNoTransactionActive); + + // second projection hooks + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willReturnCallback($expectNoHandledEvents); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectNoHandledEvents); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectOneHandledEvent); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted')->willReturnCallback($expectOneHandledEvent); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp')->willReturnCallback($expectOneHandledEvent); + + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE)->willReturnCallback($expectNoHandledEvents); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectNoHandledEvents); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willReturnCallback($expectOneHandledEvent); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted')->willReturnCallback($expectOneHandledEvent); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp')->willReturnCallback($expectOneHandledEvent); + + $expectNoHandledEvents(); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + $expectOneHandledEvent(); + } + + /** @test */ + public function catchUpBeforeAndAfterCatchupAreRunForZeroEvents() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::never())->method('apply'); + $this->subscriptionEngine->setup(); + + // first projection hooks + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + // second projection hooks + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onBeforeEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(0)); + self::assertEmpty($this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()); + } + + /** @test */ + public function catchUpBeforeAndAfterCatchupAreNotRunIfNoSubscriberMatches() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::never())->method('apply'); + $this->subscriptionEngine->setup(); + + // first projection hooks + $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeCatchUp'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterBatchCompleted'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterCatchUp'); + + // second projection hooks + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onBeforeCatchUp'); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onBeforeEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onAfterCatchUp'); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertNull($result->errors); + self::assertEquals(0, $result->numberOfProcessedEvents); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(0)); + self::assertEmpty($this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()); + } + + /** @test */ + public function catchHooksAreOnlyRunForMatchingSubscriber() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::never())->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // first projection hooks + $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeCatchUp'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterBatchCompleted'); + $this->catchupHookForFakeProjection->expects(self::never())->method('onAfterCatchUp'); + + // second projection hooks + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterBatchCompleted'); + $this->additionalCatchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $result = $this->subscriptionEngine->catchUpActive(SubscriptionEngineCriteria::create( + [SubscriptionId::fromString('Vendor.Package:SecondFakeProjection')] + )); + self::assertNull($result->errors); + self::assertEquals(1, $result->numberOfProcessedEvents); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + + public function provideValidBatchSizes(): iterable + { + yield 'none' => [ + 'batchSize' => null, + 'onAfterBatchCompletedInvocations' => [ + [1,2,3,4] + ], + ]; + yield 'one' => [ + 'batchSize' => 1, + 'onAfterBatchCompletedInvocations' => [ + [1], + [1,2], + [1,2,3], + [1,2,3,4], + [1,2,3,4], + ], + ]; + yield 'two' => [ + 'batchSize' => 2, + 'onAfterBatchCompletedInvocations' => [ + [1,2], + [1,2,3,4], + [1,2,3,4], + ], + ]; + yield 'four' => [ + 'batchSize' => 4, + // we have two calls as the batch size exactly matches the events and we are running again to see if we handled everything. + 'onAfterBatchCompletedInvocations' => [ + [1,2,3,4], + [1,2,3,4], + ], + ]; + yield 'ten' => [ + 'batchSize' => 10, + 'onAfterBatchCompletedInvocations' => [ + [1,2,3,4], + ], + ]; + } + + /** + * @dataProvider provideValidBatchSizes + * @test + */ + public function catchUpHooksWithBatching(int|null $batchSize, array $onAfterBatchCompletedInvocations) + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(4))->method('apply'); + $this->subscriptionEngine->setup(); + + // commit events (will be batched in chunks of two) + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + // first projection hooks + $this->catchupHookForFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $this->catchupHookForFakeProjection->expects(self::exactly(4))->method('onBeforeEvent'); + $this->catchupHookForFakeProjection->expects(self::exactly(4))->method('onAfterEvent'); + $this->catchupHookForFakeProjection->expects(self::exactly(\count($onAfterBatchCompletedInvocations)))->method('onAfterBatchCompleted'); + $this->catchupHookForFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + // second projection hooks + foreach ([ + $this->catchupHookForSecondFakeProjection, + $this->additionalCatchupHookForSecondFakeProjection, + ] as $catchUpHookMock) { + $catchUpHookMock->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::BOOTING); + $catchUpHookMock->expects($i = self::exactly(4))->method('onBeforeEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($i) { + match($i->getInvocationCount()) { + 1 => [ + self::assertEquals(1, $eventEnvelope->sequenceNumber->value), + self::assertEquals([], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 2 => [ + self::assertEquals(2, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 3 => [ + self::assertEquals(3, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1,2], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 4 => [ + self::assertEquals(4, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1,2,3], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + }; + }); + $catchUpHookMock->expects($i = self::exactly(4))->method('onAfterEvent')->willReturnCallback(function ($_, EventEnvelope $eventEnvelope) use ($i) { + match($i->getInvocationCount()) { + 1 => [ + self::assertEquals(1, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 2 => [ + self::assertEquals(2, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1,2], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 3 => [ + self::assertEquals(3, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1,2,3], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + 4 => [ + self::assertEquals(4, $eventEnvelope->sequenceNumber->value), + self::assertEquals([1,2,3,4], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()) + ], + }; + }); + $catchUpHookMock->expects($i = self::exactly(\count($onAfterBatchCompletedInvocations)))->method('onAfterBatchCompleted')->willReturnCallback(function () use ($i, $onAfterBatchCompletedInvocations) { + self::assertEquals($onAfterBatchCompletedInvocations[$i->getInvocationCount() - 1], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()); + }); + $catchUpHookMock->expects(self::once())->method('onAfterCatchUp'); + } + + self::assertEmpty($this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()); + + $result = $this->subscriptionEngine->boot(batchSize: $batchSize); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(4)); + self::assertEquals([1,2,3,4], $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues()); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookWithPersistenceTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookWithPersistenceTest.php new file mode 100644 index 00000000000..5acdb583039 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/CatchUpHookWithPersistenceTest.php @@ -0,0 +1,124 @@ +truncateAndSetupFlowEntities(); + } + + /** @test */ + public function commitOnConnection_onAfterEvent() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit two events. but only the first will never be seen + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->willReturnCallback(function () { + $this->getObject(Connection::class)->commit(); + }); + $this->catchupHookForSecondFakeProjection->expects(self::never())->method('onAfterCatchUp'); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $actualException = null; + try { + $this->subscriptionEngine->catchUpActive(batchSize: 1); + } catch (\Throwable $e) { + $actualException = $e; + } + // To solve this we would need to use an own connection for all CORE cr parts. + self::assertInstanceOf(\Doctrine\DBAL\ConnectionException::class, $actualException); + self::assertEquals('There is no active transaction.', $actualException->getMessage()); + + self::assertFalse($this->getObject(Connection::class)->isTransactionActive()); + + // partially applied event because the error is thrown at the end and the projection is not rolled back + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + self::assertEquals( + [1], + $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues() + ); + } + + /** @test */ + public function persistAll_onAfterEvent_willUseTheTransaction() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit one event + $this->commitExampleContentStreamEvent(); + + $persistentResource = new PersistentResource(); + $persistentResource->disableLifecycleEvents(); + $persistentResource->setFilename($expectedName = 'test_cr_catchup.empty'); + $persistentResource->setFileSize(0); + $persistentResource->setCollectionName('default'); + $persistentResource->setMediaType('text/plain'); + $persistentResource->setSha1($sha1 = '67f22467d829a254d53fa5cf019787c23c57bbef'); + + self::assertTrue($this->getObject(PersistenceManagerInterface::class)->isNewObject($persistentResource)); + + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeCatchUp')->with(SubscriptionStatus::ACTIVE); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onBeforeEvent')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterEvent')->willReturnCallback(function () use ($persistentResource) { + $this->getObject(ResourceRepository::class)->add($persistentResource); + $this->getObject(PersistenceManagerInterface::class)->persistAll(); + }); + $this->catchupHookForSecondFakeProjection->expects(self::once())->method('onAfterCatchUp'); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // check that the object was persisted and re-fetch it from the database + self::assertFalse($this->getObject(PersistenceManagerInterface::class)->isNewObject($persistentResource)); + $this->getObject(PersistenceManagerInterface::class)->clearState(); + + $actuallyPersisted = $this->getObject(ResourceRepository::class)->findOneBySha1($sha1); + + self::assertEquals($sha1, $actuallyPersisted->getSha1()); + self::assertEquals($expectedName, $actuallyPersisted->getFilename()); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/DebugEventProjectionTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/DebugEventProjectionTest.php new file mode 100644 index 00000000000..e80be3c98ea --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/DebugEventProjectionTest.php @@ -0,0 +1,107 @@ +debugEventProjection = new DebugEventProjection( + 'test_debug_projection', + Bootstrap::$staticObjectManager->get(Connection::class) + ); + + $this->debugEventProjection->setUp(); + } + + public function tearDown(): void + { + $this->debugEventProjection->resetState(); + } + + /** @test */ + public function fakeProjectionRejectsDuplicateEvents() + { + $fakeEventEnvelope = $this->createExampleEventEnvelopeForPosition( + SequenceNumber::fromInteger(1) + ); + + $this->debugEventProjection->apply( + $this->getMockBuilder(EventInterface::class)->getMock(), + $fakeEventEnvelope + ); + + $this->expectExceptionMessage('Must not happen! Debug projection detected duplicate event 1 of type ContentStreamWasCreated'); + + $this->debugEventProjection->apply( + $this->getMockBuilder(EventInterface::class)->getMock(), + $fakeEventEnvelope + ); + } + + /** @test */ + public function fakeProjectionWithSaboteur() + { + $fakeEventEnvelope1 = $this->createExampleEventEnvelopeForPosition( + SequenceNumber::fromInteger(1) + ); + + $fakeEventEnvelope2 = $this->createExampleEventEnvelopeForPosition( + SequenceNumber::fromInteger(2) + ); + + $this->debugEventProjection->injectSaboteur( + fn (EventEnvelope $eventEnvelope) => + $eventEnvelope->sequenceNumber->value === 2 + ? throw new \RuntimeException('sabotage!!!') + : null + ); + + // catchup + $this->debugEventProjection->apply( + $this->getMockBuilder(EventInterface::class)->getMock(), + $fakeEventEnvelope1 + ); + + $this->expectExceptionMessage('sabotage!!!'); + + $this->debugEventProjection->apply( + $this->getMockBuilder(EventInterface::class)->getMock(), + $fakeEventEnvelope2 + ); + } + + private function createExampleEventEnvelopeForPosition(SequenceNumber $sequenceNumber): EventEnvelope + { + $cs = ContentStreamId::create(); + return new EventEnvelope( + new Event( + Event\EventId::create(), + Event\EventType::fromString('ContentStreamWasCreated'), + Event\EventData::fromString(json_encode(['contentStreamId' => $cs->value])) + ), + ContentStreamEventStreamName::fromContentStreamId($cs)->getEventStreamName(), + Event\Version::first(), + $sequenceNumber, + new \DateTimeImmutable() + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ExternalProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ExternalProjectionErrorTest.php new file mode 100644 index 00000000000..6de1555d27e --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ExternalProjectionErrorTest.php @@ -0,0 +1,96 @@ +getObject(EntityManagerInterface::class); + + if (!isset(self::$secondConnection)) { + self::$secondConnection = DriverManager::getConnection( + $entityManager->getConnection()->getParams(), + $entityManager->getConfiguration(), + $entityManager->getEventManager() + ); + } + + $this->secondFakeProjection = new DebugEventProjection( + sprintf('cr_%s_debug_projection', self::$contentRepositoryId->value), + self::$secondConnection + ); + } + + public static function tearDownAfterClass(): void + { + self::$secondConnection->close(); + } + + /** @test */ + public function externalProjectionIsNotRolledBackAfterError() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $exception = new \RuntimeException('This projection is kaputt.'); + + $this->secondFakeProjection->injectSaboteur(fn () => throw $exception); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertSame($result->errors?->first()->message, 'This projection is kaputt.'); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // not empty as the projection is commited directly + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php new file mode 100644 index 00000000000..48973e51fe1 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/ProjectionErrorTest.php @@ -0,0 +1,330 @@ +eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::once())->method('resetState'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit an event + $this->commitExampleContentStreamEvent(); + + // catchup active tries to apply the commited event + $exception = new \RuntimeException('This projection is kaputt.'); + $this->secondFakeProjection->injectSaboteur(function (EventEnvelope $eventEnvelope) use ($exception) { + self::assertEquals(SequenceNumber::fromInteger(1), $eventEnvelope->sequenceNumber); + self::assertEquals('ContentStreamWasCreated', $eventEnvelope->event->type->value); + throw $exception; + }); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertTrue($result->hadErrors()); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + $this->secondFakeProjection->killSaboteur(); + + $result = $this->subscriptionEngine->reset(); + self::assertNull($result->errors); + + // expect the subscriptionError to be reset to null + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + + /** @test */ + public function irreparableProjection() + { + // test ways NOT to fix a projection :) + $this->eventStore->setup(); + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->fakeProjection->expects(self::once())->method('resetState'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $exception = new \RuntimeException('This projection is kaputt.'); + $this->secondFakeProjection->injectSaboteur(function (EventEnvelope $eventEnvelope) use ($exception) { + self::assertEquals(SequenceNumber::fromInteger(1), $eventEnvelope->sequenceNumber); + self::assertEquals('ContentStreamWasCreated', $eventEnvelope->event->type->value); + throw $exception; + }); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + // catchup active tries to apply the commited event + $result = $this->subscriptionEngine->catchUpActive(); + // but fails + self::assertTrue($result->hadErrors()); + self::assertEquals($expectedFailure, $this->subscriptionStatus('Vendor.Package:SecondFakeProjection')); + + // a second catchup active does not change anything + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(0), $result); + self::assertEquals($expectedFailure, $this->subscriptionStatus('Vendor.Package:SecondFakeProjection')); + + // boot neither + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + self::assertEquals($expectedFailure, $this->subscriptionStatus('Vendor.Package:SecondFakeProjection')); + + // setup neither + $result = $this->subscriptionEngine->setup(); + self::assertEquals(Result::success(), $result); + self::assertEquals($expectedFailure, $this->subscriptionStatus('Vendor.Package:SecondFakeProjection')); + + // expect the subscriptionError to be reset to null + $result = $this->subscriptionEngine->reset(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + // but booting will rethrow that error :D + $result = $this->subscriptionEngine->boot(); + self::assertTrue($result->hadErrors()); + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + // previous state is now booting + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::BOOTING, $exception), + setupStatus: ProjectionStatus::ok(), + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + } + + /** @test */ + public function projectionWithError() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $exception = new \RuntimeException('This projection is kaputt.'); + + $this->secondFakeProjection->injectSaboteur(fn () => throw $exception); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertSame($result->errors?->first()->message, 'This projection is kaputt.'); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // because the error is thrown after the even the state is commited + self::assertEquals( + [1], + $this->secondFakeProjection->getState()->findAppliedSequenceNumberValues() + ); + } + + /** @test */ + public function projectionWithErrorAfterSecondEvent() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + // commit two events + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $exception = new \RuntimeException('Event 2 is kaputt.'); + + // fail at the second event + $this->secondFakeProjection->injectSaboteur( + fn (EventEnvelope $eventEnvelope) => + $eventEnvelope->sequenceNumber->value === 2 + ? throw $exception + : null + ); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $result = $this->subscriptionEngine->catchUpActive(); + self::assertTrue($result->hadErrors()); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // the first successful event is applied and committet, but the second partially applied event is also applied: + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function projectionErrorWithMultipleProjectionsInContentRepositoryHandle() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->subscriptionEngine->setup(); + $this->subscriptionEngine->boot(); + + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( + $originalException = new \RuntimeException('This projection is kaputt.'), + ); + + $handleException = null; + try { + $this->contentRepository->handle(CreateRootWorkspace::create(WorkspaceName::fromString('root'), ContentStreamId::fromString('root-cs'))); + } catch (\RuntimeException $exception) { + $handleException = $exception; + } + self::assertInstanceOf(CatchUpHadErrors::class, $exception); + self::assertEquals('Error while catching up: Event 1 in "Vendor.Package:FakeProjection": This projection is kaputt.', $handleException->getMessage()); + self::assertSame($originalException, $handleException->getPrevious()); + + // workspace is created. The fake projection failed on the first event, but other projections succeed: + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + self::assertNotNull($this->contentRepository->findWorkspaceByName(WorkspaceName::fromString('root'))); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // to exception thrown here because the failed projection is not retried and now in error state + $this->contentRepository->handle(CreateRootWorkspace::create(WorkspaceName::fromString('root-two'), ContentStreamId::fromString('root-cs-two'))); + + // workspace two is created. + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(4)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(4)); + self::assertNotNull($this->contentRepository->findWorkspaceByName(WorkspaceName::fromString('root-two'))); + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2), SequenceNumber::fromInteger(3), SequenceNumber::fromInteger(4)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function projectionError_stopsEngineAfterFirstBatch() + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->subscriptionEngine->setup(); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + // commit two events + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class))->willThrowException( + $exception = new \RuntimeException('This projection is kaputt.') + ); + $expectedStatusForFailedProjection = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::BOOTING, $exception), + setupStatus: ProjectionStatus::ok(), + ); + + $result = $this->subscriptionEngine->boot(batchSize: 1); + self::assertEquals(ProcessedResult::failed(1, Errors::fromArray([Error::create( + SubscriptionId::fromString('Vendor.Package:FakeProjection'), + $exception->getMessage(), + $exception, + SequenceNumber::fromInteger(1) + )])), $result); + + self::assertEquals( + $expectedStatusForFailedProjection, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(1)); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php new file mode 100644 index 00000000000..421d9a4c212 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionActiveStatusTest.php @@ -0,0 +1,117 @@ +eventStore->setup(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionEngine->setup(); + + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event + $this->commitExampleContentStreamEvent(); + + // subsequent catchup setup'd does not change the position + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // catchup active does apply the commited event + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(1), $result); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + + /** @test */ + public function filteringCatchUpActive() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->eventStore->setup(); + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event: + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('apply'); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + $result = $this->subscriptionEngine->catchUpActive($filter); + self::assertNull($result->errors); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + } + + /** @test */ + public function catchupWithNoEventsKeepsThePreviousPositionOfTheSubscribers() + { + $this->eventStore->setup(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->subscriptionEngine->setup(); + + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event + $this->commitExampleContentStreamEvent(); + + // catchup active does apply the commited event + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(1), $result); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // empty catchup must keep the sequence numbers of the projections okay + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(0), $result); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBatchingTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBatchingTest.php new file mode 100644 index 00000000000..960a755cc6d --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBatchingTest.php @@ -0,0 +1,64 @@ +eventStore->setup(); + // commit three events + $this->commitExampleContentStreamEvent(); + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + $this->fakeProjection->expects(self::exactly(2))->method('apply'); + $this->subscriptionEngine->setup(); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $result = $this->subscriptionEngine->boot(batchSize: 1); + self::assertEquals(ProcessedResult::success(2), $result); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function invalidBatchSizes() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionEngine->setup(); + + $e = null; + try { + $this->subscriptionEngine->boot(batchSize: 0); + } catch (\Throwable $e) { + } + self::assertInstanceOf(\InvalidArgumentException::class, $e); + self::assertEquals(1733597950, $e->getCode()); + + try { + $this->subscriptionEngine->catchUpActive(batchSize: -1); + } catch (\Throwable $e) { + } + + self::assertInstanceOf(\InvalidArgumentException::class, $e); + self::assertEquals(1733597950, $e->getCode()); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php new file mode 100644 index 00000000000..a89370bd57e --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionBootingStatusTest.php @@ -0,0 +1,66 @@ +eventStore->setup(); + // commit an event + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionEngine->setup(); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $this->fakeProjection->expects(self::once())->method('apply')->with(self::isInstanceOf(ContentStreamWasCreated::class)); + $this->subscriptionEngine->boot(); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // catchup is a noop because there are no unhandled events + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(0), $result); + } + + /** @test */ + public function filteringCatchUpBoot() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->eventStore->setup(); + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + + $result = $this->subscriptionEngine->boot($filter); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php new file mode 100644 index 00000000000..8762343bf74 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionDetachedStatusTest.php @@ -0,0 +1,227 @@ +getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($originalSettings); + } + + /** @test */ + public function projectionIsDetachedOnCatchupActive() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->eventStore->setup(); + $this->subscriptionEngine->setup(); + + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // commit an event + $this->commitExampleContentStreamEvent(); + + $this->uninstallProjection('Vendor.Package:FakeProjection'); + + self::assertEquals( + DetachedSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + // the state is still active as we do not mutate it during the setup call! + subscriptionStatus: SubscriptionStatus::ACTIVE, + subscriptionPosition: SequenceNumber::none() + ), + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + $this->fakeProjection->expects(self::never())->method('apply'); + // catchup to mark detached subscribers + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(1), $result); + + self::assertEquals( + $expectedDetachedState = DetachedSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::DETACHED, + subscriptionPosition: SequenceNumber::none() + ), + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // other projections are not interrupted + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // succeeding catchup's do not handle the detached one: + $this->commitExampleContentStreamEvent(); + $result = $this->subscriptionEngine->catchUpActive(); + self::assertEquals(ProcessedResult::success(1), $result); + + self::assertEquals( + [SequenceNumber::fromInteger(1), SequenceNumber::fromInteger(2)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // still detached + self::assertEquals( + $expectedDetachedState, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // setup is a noop: + $result = $this->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')])); + self::assertNull($result->errors); + + // still detached + self::assertEquals( + $expectedDetachedState, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // reset is a noop: + $result = $this->subscriptionEngine->reset(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')])); + self::assertNull($result->errors); + + // still detached + self::assertEquals( + $expectedDetachedState, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + } + + /** @test */ + public function projectionIsDetachedOnSetup() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->eventStore->setup(); + $this->subscriptionEngine->setup(); + + $this->commitExampleContentStreamEvent(); + + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(1), $result); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // "uninstall" the projection + $this->uninstallProjection('Vendor.Package:FakeProjection'); + + $this->fakeProjection->expects(self::never())->method('apply'); + // setup to find detached subscribers + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + $expectedDetachedState = DetachedSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::DETACHED, + subscriptionPosition: SequenceNumber::fromInteger(1) + ); + + self::assertEquals( + $expectedDetachedState, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // "reinstall" the projection + $this->reinstallProjections(); + + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::DETACHED, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: null, + setupStatus: ProjectionStatus::ok() + ), + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + } + + /** @test */ + public function projectionIsDetachedOnSetupAndReattachedViaResetIfPossible() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::once())->method('resetState'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->eventStore->setup(); + $this->subscriptionEngine->setup(); + + $this->commitExampleContentStreamEvent(); + + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(1), $result); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // "uninstall" the projection + $this->uninstallProjection('Vendor.Package:FakeProjection'); + + $this->fakeProjection->expects(self::never())->method('apply'); + // setup to find detached subscribers + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + $expectedDetachedState = DetachedSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::DETACHED, + subscriptionPosition: SequenceNumber::fromInteger(1) + ); + + self::assertEquals( + $expectedDetachedState, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + + // "reinstall" the projection + $this->reinstallProjections(); + + // reset does re-attach the projection if its found again + $result = $this->subscriptionEngine->reset(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')])); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::fromInteger(0)); + } + + private function uninstallProjection(string $projectionName): void + { + $originalSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + $mockSettings = $originalSettings; + unset($mockSettings['contentRepositories'][$this->contentRepository->id->value]['projections'][$projectionName]); + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($mockSettings); + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); + $this->setupContentRepositoryDependencies($this->contentRepository->id); + } + + private function reinstallProjections(): void + { + $this->resetContentRepositoryRegistry(); + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); + $this->setupContentRepositoryDependencies($this->contentRepository->id); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php new file mode 100644 index 00000000000..0a624d4c08b --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionGetStatusTest.php @@ -0,0 +1,87 @@ +resetDatabase( + $this->getObject(Connection::class), + $this->contentRepository->id, + keepSchema: false + ); + + $crMaintainer = $this->getObject(ContentRepositoryRegistry::class)->buildService($this->contentRepository->id, new ContentRepositoryMaintainerFactory()); + + $status = $crMaintainer->status(); + + self::assertEquals(StatusType::SETUP_REQUIRED, $status->eventStoreStatus->type); + self::assertNull($status->eventStorePosition); + self::assertTrue($status->subscriptionStatus->isEmpty()); + + self::assertNull( + $this->subscriptionStatus('contentGraph') + ); + self::assertNull( + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + self::assertNull( + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // + // setup and fetch status + // + + // only setup content graph so that the other projections are NEW, but still found + $this->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('fake needs setup.')); + + $actualStatuses = $this->subscriptionEngine->subscriptionStatus(); + + $expected = SubscriptionStatusCollection::fromArray([ + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('contentGraph'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::ok(), + ), + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::setupRequired('fake needs setup.'), + ), + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements'), + ), + ]); + + self::assertEquals($expected, $actualStatuses); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php new file mode 100644 index 00000000000..1abce196edc --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionNewStatusTest.php @@ -0,0 +1,113 @@ +getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($originalSettings); + } + + /** @test */ + public function newProjectionIsFoundWhenConfigurationIsAdded() + { + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->eventStore->setup(); + $this->subscriptionEngine->setup(); + + $result = $this->subscriptionEngine->boot(); + self::assertEquals(ProcessedResult::success(0), $result); + + self::assertNull($this->subscriptionStatus('Vendor.Package:NewFakeProjection')); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + $newFakeProjection = $this->getMockBuilder(ProjectionInterface::class)->disableAutoReturnValueGeneration()->getMock(); + $newFakeProjection->method('getState')->willReturn(new class implements ProjectionStateInterface {}); + $newFakeProjection->expects(self::exactly(5))->method('status')->willReturnOnConsecutiveCalls( + ProjectionStatus::setupRequired('Set me up'), + ProjectionStatus::setupRequired('Set me up'), + ProjectionStatus::setupRequired('Set me up'), + ProjectionStatus::ok(), + ProjectionStatus::ok(), + ); + + FakeProjectionFactory::setProjection( + 'newFake', + $newFakeProjection + ); + + $mockSettings = $this->getObject(ConfigurationManager::class)->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepositoryRegistry'); + $mockSettings['contentRepositories'][$this->contentRepository->id->value]['projections']['Vendor.Package:NewFakeProjection'] = [ + 'factoryObjectName' => FakeProjectionFactory::class, + 'options' => [ + 'instanceId' => 'newFake' + ] + ]; + $this->getObject(ContentRepositoryRegistry::class)->injectSettings($mockSettings); + $this->getObject(ContentRepositoryRegistry::class)->resetFactoryInstance($this->contentRepository->id); + $this->setupContentRepositoryDependencies($this->contentRepository->id); + + $expectedNewState = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:NewFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::setupRequired('Set me up') + ); + + // status predicts the NEW state already (without creating it in the db) + self::assertEquals( + $expectedNewState, + $this->subscriptionStatus('Vendor.Package:NewFakeProjection') + ); + + // do something that finds new subscriptions, trigger a setup on a specific projection: + $result = $this->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); + self::assertNull($result->errors); + + self::assertEquals( + $expectedNewState, + $this->subscriptionStatus('Vendor.Package:NewFakeProjection') + ); + + // reset is a noop and skips this NEW projection! + $result = $this->subscriptionEngine->reset(SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:NewFakeProjection')])); + self::assertNull($result->errors); + // still new and NOT booting! + self::assertEquals( + $expectedNewState, + $this->subscriptionStatus('Vendor.Package:NewFakeProjection') + ); + + // setup this projection + $newFakeProjection->expects(self::once())->method('setUp'); + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:NewFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:NewFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php new file mode 100644 index 00000000000..cbebb621ceb --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionResetTest.php @@ -0,0 +1,52 @@ +fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->eventStore->setup(); + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + + // commit an event: + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::once())->method('resetState'); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + $result = $this->subscriptionEngine->reset($filter); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + self::assertEquals( + [SequenceNumber::fromInteger(1)], + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php new file mode 100644 index 00000000000..20269e80b15 --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Functional/Subscription/SubscriptionSetupTest.php @@ -0,0 +1,293 @@ +eventStore->setup(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->subscriptionEngine->setup(); + + $this->fakeProjection->expects(self::exactly(2))->method('status')->willReturn(ProjectionStatus::ok()); + $actualStatuses = $this->subscriptionEngine->subscriptionStatus(); + + $expected = SubscriptionStatusCollection::fromArray([ + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('contentGraph'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::ok(), + ), + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::ok(), + ), + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::ok(), + ), + ]); + + self::assertEquals($expected, $actualStatuses); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + self::assertEmpty( + $this->secondFakeProjection->getState()->findAppliedSequenceNumbers() + ); + } + + /** @test */ + public function filteringSetup() + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->eventStore->setup(); + + $filter = SubscriptionEngineCriteria::create([SubscriptionId::fromString('Vendor.Package:FakeProjection')]); + + $result = $this->subscriptionEngine->setup($filter); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::ok() + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + } + + /** @test */ + public function setupIsInvokedForBootingSubscribers() + { + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + // hard reset, so that the tests actually need sql migrations + $this->secondFakeProjection->dropTables(); + + $this->eventStore->setup(); + + // initial setup for FakeProjection + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + // then an update is fetched, the status changes: + + $this->secondFakeProjection->schemaNeedsAdditionalColumn('column_after_update'); + + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::BOOTING, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + } + + /** @test */ + public function setupIsInvokedForPreviouslyActiveSubscribers() + { + // Usecase: Setup a content repository and then when the subscribers are active, update to a new patch which requires a setup + + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + // hard reset, so that the tests actually need sql migrations + $this->secondFakeProjection->dropTables(); + + $this->eventStore->setup(); + // setup subscription tables + $result = $this->subscriptionEngine->setup(SubscriptionEngineCriteria::create([SubscriptionId::fromString('contentGraph')])); + self::assertNull($result->errors); + + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + // initial setup for FakeProjection + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // regular work + + $this->commitExampleContentStreamEvent(); + $result = $this->subscriptionEngine->catchUpActive(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // then an update is fetched, the status changes: + + $this->secondFakeProjection->schemaNeedsAdditionalColumn('column_after_update'); + + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ACTIVE, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: null, + setupStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + + /** @test */ + public function failingSetupWillMarkProjectionAsErrored() + { + $this->fakeProjection->expects(self::once())->method('setUp')->willThrowException( + $exception = new \RuntimeException('Projection could not be setup') + ); + $this->fakeProjection->expects(self::once())->method('status')->willReturn(ProjectionStatus::setupRequired('Needs setup')); + + $this->eventStore->setup(); + + $result = $this->subscriptionEngine->setup(); + self::assertSame('Projection could not be setup', $result->errors?->first()->message); + + $expectedFailure = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:FakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::NEW, $exception), + setupStatus: ProjectionStatus::setupRequired('Needs setup'), + ); + + self::assertEquals( + $expectedFailure, + $this->subscriptionStatus('Vendor.Package:FakeProjection') + ); + } + + /** @test */ + public function failingSetupWillNotRollbackProjection() + { + // we cannot wrap the schema creation in transactions as CREATE TABLE would for example lead to an implicit commit + // and cannot be rolled back: https://dev.mysql.com/doc/refman/8.0/en/implicit-commit.html + + $this->fakeProjection->expects(self::exactly(2))->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + // hard reset, so that the tests actually need sql migrations + $this->secondFakeProjection->dropTables(); + $this->eventStore->setup(); + + // initial setup for FakeProjection + $result = $this->subscriptionEngine->setup(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $result = $this->subscriptionEngine->boot(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + // regular work + $this->commitExampleContentStreamEvent(); + $result = $this->subscriptionEngine->catchUpActive(); + self::assertNull($result->errors); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + + // then an update is fetched - but the migration contains an error: + $this->secondFakeProjection->schemaNeedsAdditionalColumn('column_after_update'); + + $exception = new \RuntimeException('Setup failed after it did some sql queries!'); + $this->secondFakeProjection->injectSaboteur(fn () => throw $exception); + + self::assertEquals( + ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ACTIVE, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: null, + setupStatus: ProjectionStatus::setupRequired('Requires 1 SQL statements') + ), + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + + self::assertNull($result->errors); + + $expectedStatusForFailedProjection = ProjectionSubscriptionStatus::create( + subscriptionId: SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + subscriptionStatus: SubscriptionStatus::ERROR, + subscriptionPosition: SequenceNumber::fromInteger(1), + subscriptionError: SubscriptionError::fromPreviousStatusAndException(SubscriptionStatus::ACTIVE, $exception), + // as we cant roll back, the migration was (possibly partially) made: + setupStatus: ProjectionStatus::ok() + ); + + $result = $this->subscriptionEngine->setup(); + self::assertEquals(Errors::fromArray([Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $exception->getMessage(), + $exception, + null + )]), $result->errors); + + self::assertEquals( + $expectedStatusForFailedProjection, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection') + ); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php index 569609ee5ce..efa68008845 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/AbstractParallelTestCase.php @@ -14,8 +14,9 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Parallel; -use Doctrine\DBAL\Connection; use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\Flow\Core\Bootstrap; @@ -69,15 +70,11 @@ final protected function setUpContentRepository( ContentRepositoryId $contentRepositoryId ): ContentRepository { $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); - $contentRepository->setUp(); - - $connection = $this->objectManager->get(Connection::class); - + /** @var ContentRepositoryMaintainer $contentRepositoryMaintainer */ + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + $contentRepositoryMaintainer->setUp(); // reset events and projections - $eventTableName = sprintf('cr_%s_events', $contentRepositoryId->value); - $connection->executeStatement('TRUNCATE ' . $eventTableName); - $contentRepository->resetProjectionStates(); - + $contentRepositoryMaintainer->prune(); return $contentRepository; } diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php new file mode 100644 index 00000000000..06b508c094e --- /dev/null +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/ParallelWritingInWorkspaces/ParallelWritingInWorkspacesTest.php @@ -0,0 +1,227 @@ +log('------ process started ------'); + + $debugProjection = new DebugEventProjection( + 'cr_test_parallel_debug_projection', + $this->objectManager->get(Connection::class) + ); + FakeProjectionFactory::setProjection( + 'debug', + $debugProjection + ); + + FakeContentDimensionSourceFactory::setWithoutDimensions(); + FakeNodeTypeManagerFactory::setConfiguration([ + 'Neos.ContentRepository:Root' => [], + 'Neos.ContentRepository.Testing:Content' => [], + 'Neos.ContentRepository.Testing:Document' => [ + 'properties' => [ + 'title' => [ + 'type' => 'string' + ] + ], + 'childNodes' => [ + 'tethered-a' => [ + 'type' => 'Neos.ContentRepository.Testing:Content' + ], + 'tethered-b' => [ + 'type' => 'Neos.ContentRepository.Testing:Content' + ], + 'tethered-c' => [ + 'type' => 'Neos.ContentRepository.Testing:Content' + ], + 'tethered-d' => [ + 'type' => 'Neos.ContentRepository.Testing:Content' + ], + 'tethered-e' => [ + 'type' => 'Neos.ContentRepository.Testing:Content' + ] + ] + ] + ]); + + $setupLockResource = fopen(self::SETUP_LOCK_PATH, 'w+'); + + $exclusiveNonBlockingLockResult = flock($setupLockResource, LOCK_EX | LOCK_NB); + if ($exclusiveNonBlockingLockResult === false) { + $this->log('waiting for setup'); + if (!flock($setupLockResource, LOCK_SH)) { + throw new \RuntimeException('failed to acquire blocking shared lock'); + } + $this->contentRepository = $this->contentRepositoryRegistry + ->get(ContentRepositoryId::fromString('test_parallel')); + $this->log('wait for setup finished'); + return; + } + + $this->log('setup started'); + $contentRepository = $this->setUpContentRepository(ContentRepositoryId::fromString('test_parallel')); + + $origin = OriginDimensionSpacePoint::createWithoutDimensions(); + $contentRepository->handle(CreateRootWorkspace::create( + WorkspaceName::forLive(), + ContentStreamId::fromString('live-cs-id') + )); + $contentRepository->handle(CreateRootNodeAggregateWithNode::create( + WorkspaceName::forLive(), + NodeAggregateId::fromString('lady-eleonode-rootford'), + NodeTypeName::fromString(NodeTypeName::ROOT_NODE_TYPE_NAME) + )); + $contentRepository->handle(CreateNodeAggregateWithNode::create( + WorkspaceName::forLive(), + NodeAggregateId::fromString('nody-mc-nodeface'), + NodeTypeName::fromString('Neos.ContentRepository.Testing:Document'), + $origin, + NodeAggregateId::fromString('lady-eleonode-rootford'), + initialPropertyValues: PropertyValuesToWrite::fromArray([ + 'title' => 'title-original' + ]) + )); + $contentRepository->handle(CreateWorkspace::create( + WorkspaceName::fromString('user-test'), + WorkspaceName::forLive(), + ContentStreamId::fromString('user-cs-id') + )); + + $this->contentRepository = $contentRepository; + + if (!flock($setupLockResource, LOCK_UN)) { + throw new \RuntimeException('failed to release setup lock'); + } + + $this->log('setup finished'); + } + + /** + * @test + * @group parallel + */ + public function whileANodesArWrittenOnLive(): void + { + $this->log('1. writing started'); + + touch(self::WRITING_IS_RUNNING_FLAG_PATH); + + try { + for ($i = 0; $i <= 100; $i++) { + $this->contentRepository->handle(CreateNodeAggregateWithNode::create( + WorkspaceName::forLive(), + NodeAggregateId::fromString('nody-mc-nodeface-' . $i), + NodeTypeName::fromString('Neos.ContentRepository.Testing:Document'), + OriginDimensionSpacePoint::createWithoutDimensions(), + NodeAggregateId::fromString('lady-eleonode-rootford'), + initialPropertyValues: PropertyValuesToWrite::fromArray([ + 'title' => 'title' + ]) + )); + } + } finally { + unlink(self::WRITING_IS_RUNNING_FLAG_PATH); + } + + $this->log('1. writing finished'); + Assert::assertTrue(true, 'No exception was thrown ;)'); + + $subgraph = $this->contentRepository->getContentGraph(WorkspaceName::forLive())->getSubgraph(DimensionSpacePoint::createWithoutDimensions(), VisibilityConstraints::withoutRestrictions()); + $node = $subgraph->findNodeById(NodeAggregateId::fromString('nody-mc-nodeface-100')); + Assert::assertNotNull($node); + } + + /** + * @test + * @group parallel + */ + public function thenConcurrentPublishLeadsToException(): void + { + if (!is_file(self::WRITING_IS_RUNNING_FLAG_PATH)) { + $this->log('waiting for 2. writing'); + + $this->awaitFile(self::WRITING_IS_RUNNING_FLAG_PATH); + // If write is the process that does the (slowish) setup, and then waits for the rebase to start, + // We give the CR some time to close the content stream + // TODO find another way than to randomly wait!!! + // The problem is, if we dont sleep it happens often that the modification works only then the rebase is startet _really_ + // Doing the modification several times in hope that the second one fails will likely just stop the rebase thread as it cannot close + usleep(10000); + } + + $this->log('2. writing started'); + + for ($i = 0; $i <= 100; $i++) { + $this->contentRepository->handle(CreateNodeAggregateWithNode::create( + WorkspaceName::fromString('user-test'), + NodeAggregateId::fromString('user-nody-mc-nodeface-' . $i), + NodeTypeName::fromString('Neos.ContentRepository.Testing:Document'), + OriginDimensionSpacePoint::createWithoutDimensions(), + NodeAggregateId::fromString('lady-eleonode-rootford'), + initialPropertyValues: PropertyValuesToWrite::fromArray([ + 'title' => 'title' + ]) + )); + } + + $this->log('2. writing finished'); + + Assert::assertTrue(true, 'No exception was thrown ;)'); + + $subgraph = $this->contentRepository->getContentGraph(WorkspaceName::fromString('user-test'))->getSubgraph(DimensionSpacePoint::createWithoutDimensions(), VisibilityConstraints::withoutRestrictions()); + $node = $subgraph->findNodeById(NodeAggregateId::fromString('user-nody-mc-nodeface-100')); + Assert::assertNotNull($node); + } +} diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php index 1dcc9fcdd03..3c62dcc5149 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspacePublicationDuringWriting/WorkspacePublicationDuringWritingTest.php @@ -14,7 +14,9 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Parallel\WorkspacePublicationDuringWriting; +use Doctrine\DBAL\Connection; use Neos\ContentRepository\BehavioralTests\Tests\Parallel\AbstractParallelTestCase; +use Neos\ContentRepository\BehavioralTests\TestSuite\DebugEventProjection; use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePoint; use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint; @@ -34,6 +36,7 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory; use Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory; +use Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory; use Neos\EventStore\Exception\ConcurrencyException; use Neos\Flow\ObjectManagement\ObjectManagerInterface; use PHPUnit\Framework\Assert; @@ -51,6 +54,16 @@ public function setUp(): void { parent::setUp(); $this->log('------ process started ------'); + + $debugProjection = new DebugEventProjection( + 'cr_test_parallel_debug_projection', + $this->objectManager->get(Connection::class) + ); + FakeProjectionFactory::setProjection( + 'debug', + $debugProjection + ); + FakeContentDimensionSourceFactory::setWithoutDimensions(); FakeNodeTypeManagerFactory::setConfiguration([ 'Neos.ContentRepository:Root' => [], @@ -155,6 +168,11 @@ public function whileANodesArWrittenOnLive(): void $this->log('writing finished'); Assert::assertTrue(true, 'No exception was thrown ;)'); + + $subgraph = $this->contentRepository->getContentGraph(WorkspaceName::forLive())->getSubgraph(DimensionSpacePoint::createWithoutDimensions(), VisibilityConstraints::withoutRestrictions()); + $node = $subgraph->findNodeById(NodeAggregateId::fromString('nody-mc-nodeface')); + Assert::assertNotNull($node); + Assert::assertSame($node->getProperty('title'), 'changed-title-50'); } /** diff --git a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspaceWritingDuringRebase/WorkspaceWritingDuringRebaseTest.php b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspaceWritingDuringRebase/WorkspaceWritingDuringRebaseTest.php index f4a37360ed1..566a86c9bc9 100644 --- a/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspaceWritingDuringRebase/WorkspaceWritingDuringRebaseTest.php +++ b/Neos.ContentRepository.BehavioralTests/Tests/Parallel/WorkspaceWritingDuringRebase/WorkspaceWritingDuringRebaseTest.php @@ -14,7 +14,9 @@ namespace Neos\ContentRepository\BehavioralTests\Tests\Parallel\WorkspaceWritingDuringRebase; +use Doctrine\DBAL\Connection; use Neos\ContentRepository\BehavioralTests\Tests\Parallel\AbstractParallelTestCase; +use Neos\ContentRepository\BehavioralTests\TestSuite\DebugEventProjection; use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePoint; use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint; @@ -35,6 +37,7 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory; use Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory; +use Neos\ContentRepository\TestSuite\Fakes\FakeProjectionFactory; use Neos\EventStore\Exception\ConcurrencyException; use Neos\Flow\ObjectManagement\ObjectManagerInterface; use PHPUnit\Framework\Assert; @@ -53,6 +56,16 @@ public function setUp(): void { parent::setUp(); $this->log('------ process started ------'); + + $debugProjection = new DebugEventProjection( + 'cr_test_parallel_debug_projection', + $this->objectManager->get(Connection::class) + ); + FakeProjectionFactory::setProjection( + 'debug', + $debugProjection + ); + FakeContentDimensionSourceFactory::setWithoutDimensions(); FakeNodeTypeManagerFactory::setConfiguration([ 'Neos.ContentRepository:Root' => [], diff --git a/Neos.ContentRepository.Core/Classes/CommandHandler/CommandSimulator.php b/Neos.ContentRepository.Core/Classes/CommandHandler/CommandSimulator.php index 9becebc5a2d..f004613778f 100644 --- a/Neos.ContentRepository.Core/Classes/CommandHandler/CommandSimulator.php +++ b/Neos.ContentRepository.Core/Classes/CommandHandler/CommandSimulator.php @@ -135,11 +135,6 @@ private function handle(RebaseableCommand $rebaseableCommand): void foreach ($eventStream as $eventEnvelope) { $event = $this->eventNormalizer->denormalize($eventEnvelope->event); - - if (!$this->contentRepositoryProjection->canHandle($event)) { - continue; - } - $this->contentRepositoryProjection->apply($event, $eventEnvelope); } } diff --git a/Neos.ContentRepository.Core/Classes/CommandHandler/Commands.php b/Neos.ContentRepository.Core/Classes/CommandHandler/Commands.php index c76eb1955e7..85cee65f834 100644 --- a/Neos.ContentRepository.Core/Classes/CommandHandler/Commands.php +++ b/Neos.ContentRepository.Core/Classes/CommandHandler/Commands.php @@ -4,6 +4,8 @@ namespace Neos\ContentRepository\Core\CommandHandler; +use Neos\ContentRepository\Core\Subscription\Exception\CatchUpHadErrors; + /** * @api can be used as collection of commands to be individually handled: * @@ -11,6 +13,9 @@ * $contentRepository->handle($command); * } * + * Note that as they are separate commands, they might individually fail due to constraints + * or a projection or catchup failing during the first catchup with {@see CatchUpHadErrors} + * * @implements \IteratorAggregate */ final readonly class Commands implements \IteratorAggregate, \Countable diff --git a/Neos.ContentRepository.Core/Classes/ContentRepository.php b/Neos.ContentRepository.Core/Classes/ContentRepository.php index 556ba4379be..f308279ac42 100644 --- a/Neos.ContentRepository.Core/Classes/ContentRepository.php +++ b/Neos.ContentRepository.Core/Classes/ContentRepository.php @@ -21,27 +21,18 @@ use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePoint; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; use Neos\ContentRepository\Core\EventStore\EventNormalizer; -use Neos\ContentRepository\Core\EventStore\EventPersister; use Neos\ContentRepository\Core\EventStore\EventsToPublish; use Neos\ContentRepository\Core\EventStore\InitiatingEventMetadata; use Neos\ContentRepository\Core\Feature\Security\AuthProviderInterface; use Neos\ContentRepository\Core\Feature\Security\Dto\UserId; use Neos\ContentRepository\Core\Feature\Security\Exception\AccessDenied; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; -use Neos\ContentRepository\Core\Projection\CatchUp; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpOptions; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphInterface; -use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentSubgraphInterface; -use Neos\ContentRepository\Core\Projection\ProjectionInterface; -use Neos\ContentRepository\Core\Projection\ProjectionsAndCatchUpHooks; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; -use Neos\ContentRepository\Core\Projection\ProjectionStatuses; -use Neos\ContentRepository\Core\Projection\WithMarkStaleInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStates; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; -use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryStatus; use Neos\ContentRepository\Core\SharedModel\Exception\WorkspaceDoesNotExist; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStream; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; @@ -49,30 +40,24 @@ use Neos\ContentRepository\Core\SharedModel\Workspace\Workspace; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Core\SharedModel\Workspace\Workspaces; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; +use Neos\ContentRepository\Core\Subscription\Exception\CatchUpHadErrors; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Exception\ConcurrencyException; -use Neos\EventStore\Model\EventEnvelope; -use Neos\EventStore\Model\EventStream\VirtualStreamName; use Psr\Clock\ClockInterface; /** * Main Entry Point to the system. Encapsulates the full event-sourced Content Repository. * * Use this to: - * - set up the necessary database tables and contents via {@see ContentRepository::setUp()} - * - send commands to the system (to mutate state) via {@see ContentRepository::handle()} - * - access projection state (to read state) via {@see ContentRepository::projectionState()} - * - catch up projections via {@see ContentRepository::catchUpProjection()} + * - send commands to the system (to mutate state) via {@see self::handle()} + * - access the content graph read model + * - access 3rd party read models via {@see self::projectionState()} * * @api */ final class ContentRepository { - /** - * @var array, ProjectionStateInterface> - */ - private array $projectionStateCache; - /** * @internal use the {@see ContentRepositoryFactory::getOrBuild()} to instantiate */ @@ -80,9 +65,8 @@ public function __construct( public readonly ContentRepositoryId $id, private readonly CommandBus $commandBus, private readonly EventStoreInterface $eventStore, - private readonly ProjectionsAndCatchUpHooks $projectionsAndCatchUpHooks, private readonly EventNormalizer $eventNormalizer, - private readonly EventPersister $eventPersister, + private readonly SubscriptionEngine $subscriptionEngine, private readonly NodeTypeManager $nodeTypeManager, private readonly InterDimensionalVariationGraph $variationGraph, private readonly ContentDimensionSourceInterface $contentDimensionSource, @@ -90,6 +74,7 @@ public function __construct( private readonly ClockInterface $clock, private readonly ContentGraphReadModelInterface $contentGraphReadModel, private readonly CommandHookInterface $commandHook, + private readonly ProjectionStates $projectionStates, ) { } @@ -112,8 +97,11 @@ public function handle(CommandInterface $command): void // simple case if ($toPublish instanceof EventsToPublish) { $eventsToPublish = $this->enrichEventsToPublishWithMetadata($toPublish); - $this->eventPersister->publishWithoutCatchup($eventsToPublish); - $this->catchupProjections(); + $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($eventsToPublish->events), $eventsToPublish->expectedVersion); + $fullCatchUpResult = $this->subscriptionEngine->catchUpActive(); // NOTE: we don't batch here, to ensure the catchup is run completely and any errors don't stop it. + if ($fullCatchUpResult->hadErrors()) { + throw CatchUpHadErrors::createFromErrors($fullCatchUpResult->errors); + } return; } @@ -122,19 +110,19 @@ public function handle(CommandInterface $command): void foreach ($toPublish as $yieldedEventsToPublish) { $eventsToPublish = $this->enrichEventsToPublishWithMetadata($yieldedEventsToPublish); try { - $this->eventPersister->publishWithoutCatchup($eventsToPublish); + $this->eventStore->commit($eventsToPublish->streamName, $this->eventNormalizer->normalizeEvents($eventsToPublish->events), $eventsToPublish->expectedVersion); } catch (ConcurrencyException $concurrencyException) { // we pass the exception into the generator (->throw), so it could be try-caught and reacted upon: // // try { - // yield EventsToPublish(...); + // yield new EventsToPublish(...); // } catch (ConcurrencyException $e) { // yield $this->reopenContentStream(); // throw $e; // } $yieldedErrorStrategy = $toPublish->throw($concurrencyException); if ($yieldedErrorStrategy instanceof EventsToPublish) { - $this->eventPersister->publishWithoutCatchup($yieldedErrorStrategy); + $this->eventStore->commit($yieldedErrorStrategy->streamName, $this->eventNormalizer->normalizeEvents($yieldedErrorStrategy->events), $yieldedErrorStrategy->expectedVersion); } throw $concurrencyException; } @@ -142,7 +130,10 @@ public function handle(CommandInterface $command): void } finally { // We always NEED to catchup even if there was an unexpected ConcurrencyException to make sure previous commits are handled. // Technically it would be acceptable for the catchup to fail here (due to hook errors) because all the events are already persisted. - $this->catchupProjections(); + $fullCatchUpResult = $this->subscriptionEngine->catchUpActive(); // NOTE: we don't batch here, to ensure the catchup is run completely and any errors don't stop it. + if ($fullCatchUpResult->hadErrors()) { + throw CatchUpHadErrors::createFromErrors($fullCatchUpResult->errors); + } } } @@ -154,119 +145,7 @@ public function handle(CommandInterface $command): void */ public function projectionState(string $projectionStateClassName): ProjectionStateInterface { - if (!isset($this->projectionStateCache)) { - foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { - if ($projection instanceof ContentGraphProjectionInterface) { - continue; - } - $projectionState = $projection->getState(); - $this->projectionStateCache[$projectionState::class] = $projectionState; - } - } - if (isset($this->projectionStateCache[$projectionStateClassName])) { - /** @var T $projectionState */ - $projectionState = $this->projectionStateCache[$projectionStateClassName]; - return $projectionState; - } - if (in_array(ContentGraphReadModelInterface::class, class_implements($projectionStateClassName), true)) { - throw new \InvalidArgumentException(sprintf('Accessing the internal content repository projection state via %s(%s) is not allowed. Please use the API on the content repository instead.', __FUNCTION__, $projectionStateClassName), 1729338679); - } - - throw new \InvalidArgumentException(sprintf('A projection state of type "%s" is not registered in this content repository instance.', $projectionStateClassName), 1662033650); - } - - /** - * @param class-string> $projectionClassName - */ - public function catchUpProjection(string $projectionClassName, CatchUpOptions $options): void - { - $projection = $this->projectionsAndCatchUpHooks->projections->get($projectionClassName); - - $catchUpHookFactory = $this->projectionsAndCatchUpHooks->getCatchUpHookFactoryForProjection($projection); - $catchUpHook = $catchUpHookFactory?->build(CatchUpHookFactoryDependencies::create( - $this->id, - $projection->getState(), - $this->nodeTypeManager, - $this->contentDimensionSource, - $this->variationGraph - )); - - // TODO allow custom stream name per projection - $streamName = VirtualStreamName::all(); - $eventStream = $this->eventStore->load($streamName); - if ($options->maximumSequenceNumber !== null) { - $eventStream = $eventStream->withMaximumSequenceNumber($options->maximumSequenceNumber); - } - - $eventApplier = function (EventEnvelope $eventEnvelope) use ($projection, $catchUpHook, $options) { - $event = $this->eventNormalizer->denormalize($eventEnvelope->event); - if ($options->progressCallback !== null) { - ($options->progressCallback)($event, $eventEnvelope); - } - if (!$projection->canHandle($event)) { - return; - } - $catchUpHook?->onBeforeEvent($event, $eventEnvelope); - $projection->apply($event, $eventEnvelope); - if ($projection instanceof WithMarkStaleInterface) { - $projection->markStale(); - } - $catchUpHook?->onAfterEvent($event, $eventEnvelope); - }; - - $catchUp = CatchUp::create($eventApplier, $projection->getCheckpointStorage()); - - if ($catchUpHook !== null) { - $catchUpHook->onBeforeCatchUp(); - $catchUp = $catchUp->withOnBeforeBatchCompleted(fn() => $catchUpHook->onBeforeBatchCompleted()); - } - $catchUp->run($eventStream); - $catchUpHook?->onAfterCatchUp(); - } - - public function catchupProjections(): void - { - foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { - // FIXME optimise by only loading required events once and not per projection - // see https://github.com/neos/neos-development-collection/pull/4988/ - $this->catchUpProjection($projection::class, CatchUpOptions::create()); - } - } - - public function setUp(): void - { - $this->eventStore->setup(); - foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { - $projection->setUp(); - } - } - - public function status(): ContentRepositoryStatus - { - $projectionStatuses = ProjectionStatuses::createEmpty(); - foreach ($this->projectionsAndCatchUpHooks->projections as $projectionClassName => $projection) { - $projectionStatuses = $projectionStatuses->with($projectionClassName, $projection->status()); - } - return new ContentRepositoryStatus( - $this->eventStore->status(), - $projectionStatuses, - ); - } - - public function resetProjectionStates(): void - { - foreach ($this->projectionsAndCatchUpHooks->projections as $projection) { - $projection->reset(); - } - } - - /** - * @param class-string> $projectionClassName - */ - public function resetProjectionState(string $projectionClassName): void - { - $projection = $this->projectionsAndCatchUpHooks->projections->get($projectionClassName); - $projection->reset(); + return $this->projectionStates->get($projectionStateClassName); } /** diff --git a/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php b/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php index 6a7ca968917..6cb9bdd474b 100644 --- a/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php +++ b/Neos.ContentRepository.Core/Classes/EventStore/EventNormalizer.php @@ -4,7 +4,7 @@ namespace Neos\ContentRepository\Core\EventStore; -use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\EventStore\Events as DomainEvents; use Neos\ContentRepository\Core\Feature\ContentStreamClosing\Event\ContentStreamWasClosed; use Neos\ContentRepository\Core\Feature\ContentStreamClosing\Event\ContentStreamWasReopened; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; @@ -41,37 +41,38 @@ use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasPublished; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceRebaseFailed; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; -use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\EventStore\Model\Event; use Neos\EventStore\Model\Event\EventData; use Neos\EventStore\Model\Event\EventId; use Neos\EventStore\Model\Event\EventType; +use Neos\EventStore\Model\Events; /** * Central authority to convert Content Repository domain events to Event Store EventData and EventType, vice versa. * - * For normalizing (from classes to event store), this is called from {@see ContentRepository::normalizeEvent()}. + * - For normalizing (from classes to event store) + * - For denormalizing (from event store to classes) * - * For denormalizing (from event store to classes), this is called in the individual projections; f.e. - * {@see ProjectionInterface::apply()}. - * - * @api because inside projections, you get an instance of EventNormalizer to handle events. + * @internal inside projections the event will already be denormalized */ -final class EventNormalizer +final readonly class EventNormalizer { - /** - * @var array,EventType> - */ - private array $fullClassNameToShortEventType = []; - /** - * @var array> - */ - private array $shortEventTypeToFullClassName = []; + private function __construct( + /** + * @var array,EventType> + */ + private array $fullClassNameToShortEventType, + /** + * @var array> + */ + private array $shortEventTypeToFullClassName, + ) { + } /** - * @internal never instanciate this object yourself + * @internal never instantiate this object yourself */ - public function __construct() + public static function create(): self { $supportedEventClassNames = [ ContentStreamWasClosed::class, @@ -111,12 +112,20 @@ public function __construct() WorkspaceBaseWorkspaceWasChanged::class, ]; + $fullClassNameToShortEventType = []; + $shortEventTypeToFullClassName = []; + foreach ($supportedEventClassNames as $fullEventClassName) { $shortEventClassName = substr($fullEventClassName, strrpos($fullEventClassName, '\\') + 1); - $this->fullClassNameToShortEventType[$fullEventClassName] = EventType::fromString($shortEventClassName); - $this->shortEventTypeToFullClassName[$shortEventClassName] = $fullEventClassName; + $fullClassNameToShortEventType[$fullEventClassName] = EventType::fromString($shortEventClassName); + $shortEventTypeToFullClassName[$shortEventClassName] = $fullEventClassName; } + + return new self( + fullClassNameToShortEventType: $fullClassNameToShortEventType, + shortEventTypeToFullClassName: $shortEventTypeToFullClassName + ); } /** @@ -147,6 +156,11 @@ public function normalize(EventInterface|DecoratedEvent $event): Event ); } + public function normalizeEvents(DomainEvents $events): Events + { + return Events::fromArray($events->map($this->normalize(...))); + } + public function denormalize(Event $event): EventInterface { $eventClassName = $this->getEventClassName($event); diff --git a/Neos.ContentRepository.Core/Classes/EventStore/EventPersister.php b/Neos.ContentRepository.Core/Classes/EventStore/EventPersister.php deleted file mode 100644 index 1af59ff3ce9..00000000000 --- a/Neos.ContentRepository.Core/Classes/EventStore/EventPersister.php +++ /dev/null @@ -1,52 +0,0 @@ -publishWithoutCatchup($eventsToPublish); - $contentRepository->catchUpProjections(); - } - - /** - * TODO Will be refactored via https://github.com/neos/neos-development-collection/pull/5321 - * @throws ConcurrencyException in case the expectedVersion does not match - */ - public function publishWithoutCatchup(EventsToPublish $eventsToPublish): CommitResult - { - $normalizedEvents = Events::fromArray( - $eventsToPublish->events->map($this->eventNormalizer->normalize(...)) - ); - return $this->eventStore->commit( - $eventsToPublish->streamName, - $normalizedEvents, - $eventsToPublish->expectedVersion - ); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php index 09c51da4293..fde8bdd29a7 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryFactory.php @@ -22,18 +22,28 @@ use Neos\ContentRepository\Core\DimensionSpace\ContentDimensionZookeeper; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; use Neos\ContentRepository\Core\EventStore\EventNormalizer; -use Neos\ContentRepository\Core\EventStore\EventPersister; use Neos\ContentRepository\Core\Feature\DimensionSpaceAdjustment\DimensionSpaceCommandHandler; use Neos\ContentRepository\Core\Feature\NodeAggregateCommandHandler; use Neos\ContentRepository\Core\Feature\NodeDuplication\NodeDuplicationCommandHandler; use Neos\ContentRepository\Core\Feature\WorkspaceCommandHandler; use Neos\ContentRepository\Core\Infrastructure\Property\PropertyConverter; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; -use Neos\ContentRepository\Core\Projection\ProjectionsAndCatchUpHooks; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; +use Neos\ContentRepository\Core\Projection\ProjectionStates; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; +use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; +use Neos\ContentRepository\Core\Subscription\Subscriber\ProjectionSubscriber; +use Neos\ContentRepository\Core\Subscription\Subscriber\Subscribers; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepositoryRegistry\Factory\AuthProvider\AuthProviderFactoryInterface; use Neos\EventStore\EventStoreInterface; use Psr\Clock\ClockInterface; +use Psr\Log\LoggerInterface; use Symfony\Component\Serializer\Serializer; /** @@ -43,44 +53,79 @@ */ final class ContentRepositoryFactory { - private ProjectionFactoryDependencies $projectionFactoryDependencies; - private ProjectionsAndCatchUpHooks $projectionsAndCatchUpHooks; + private SubscriptionEngine $subscriptionEngine; + private ContentGraphProjectionInterface $contentGraphProjection; + private ProjectionStates $additionalProjectionStates; + private EventNormalizer $eventNormalizer; + private ContentDimensionZookeeper $contentDimensionZookeeper; + private InterDimensionalVariationGraph $interDimensionalVariationGraph; + private PropertyConverter $propertyConverter; + // guards against recursion and memory overflow + private bool $isBuilding = false; + + // The "singleton" reference for this content repository + private ?ContentRepository $contentRepositoryRuntimeCache = null; + + /** + * @param CatchUpHookFactoryInterface|null $contentGraphCatchUpHookFactory + */ public function __construct( private readonly ContentRepositoryId $contentRepositoryId, - EventStoreInterface $eventStore, - NodeTypeManager $nodeTypeManager, - ContentDimensionSourceInterface $contentDimensionSource, + private readonly EventStoreInterface $eventStore, + private readonly NodeTypeManager $nodeTypeManager, + private readonly ContentDimensionSourceInterface $contentDimensionSource, Serializer $propertySerializer, - ProjectionsAndCatchUpHooksFactory $projectionsAndCatchUpHooksFactory, private readonly AuthProviderFactoryInterface $authProviderFactory, private readonly ClockInterface $clock, + SubscriptionStoreInterface $subscriptionStore, + ContentGraphProjectionFactoryInterface $contentGraphProjectionFactory, + private readonly CatchUpHookFactoryInterface|null $contentGraphCatchUpHookFactory, private readonly CommandHooksFactory $commandHooksFactory, + private readonly ContentRepositorySubscriberFactories $additionalSubscriberFactories, + LoggerInterface|null $logger = null, ) { - $contentDimensionZookeeper = new ContentDimensionZookeeper($contentDimensionSource); - $interDimensionalVariationGraph = new InterDimensionalVariationGraph( + $this->contentDimensionZookeeper = new ContentDimensionZookeeper($contentDimensionSource); + $this->interDimensionalVariationGraph = new InterDimensionalVariationGraph( $contentDimensionSource, - $contentDimensionZookeeper + $this->contentDimensionZookeeper ); - $this->projectionFactoryDependencies = new ProjectionFactoryDependencies( + $this->eventNormalizer = EventNormalizer::create(); + $this->propertyConverter = new PropertyConverter($propertySerializer); + $subscriberFactoryDependencies = SubscriberFactoryDependencies::create( $contentRepositoryId, - $eventStore, - new EventNormalizer(), $nodeTypeManager, $contentDimensionSource, - $contentDimensionZookeeper, - $interDimensionalVariationGraph, - new PropertyConverter($propertySerializer), + $this->interDimensionalVariationGraph, + $this->propertyConverter, ); - $this->projectionsAndCatchUpHooks = $projectionsAndCatchUpHooksFactory->build($this->projectionFactoryDependencies); + $subscribers = []; + $additionalProjectionStates = []; + foreach ($this->additionalSubscriberFactories as $additionalSubscriberFactory) { + $subscriber = $additionalSubscriberFactory->build($subscriberFactoryDependencies); + $additionalProjectionStates[] = $subscriber->projection->getState(); + $subscribers[] = $subscriber; + } + $this->additionalProjectionStates = ProjectionStates::fromArray($additionalProjectionStates); + $this->contentGraphProjection = $contentGraphProjectionFactory->build($subscriberFactoryDependencies); + $subscribers[] = $this->buildContentGraphSubscriber(); + $this->subscriptionEngine = new SubscriptionEngine($this->eventStore, $subscriptionStore, Subscribers::fromArray($subscribers), $this->eventNormalizer, $logger); } - // guards against recursion and memory overflow - private bool $isBuilding = false; - - // The following properties store "singleton" references of objects for this content repository - private ?ContentRepository $contentRepository = null; - private ?EventPersister $eventPersister = null; + private function buildContentGraphSubscriber(): ProjectionSubscriber + { + return new ProjectionSubscriber( + SubscriptionId::fromString('contentGraph'), + $this->contentGraphProjection, + $this->contentGraphCatchUpHookFactory?->build(CatchUpHookFactoryDependencies::create( + $this->contentRepositoryId, + $this->contentGraphProjection->getState(), + $this->nodeTypeManager, + $this->contentDimensionSource, + $this->interDimensionalVariationGraph, + )), + ); + } /** * Builds and returns the content repository. If it is already built, returns the same instance. @@ -90,75 +135,75 @@ public function __construct( */ public function getOrBuild(): ContentRepository { - if ($this->contentRepository) { - return $this->contentRepository; + if ($this->contentRepositoryRuntimeCache) { + return $this->contentRepositoryRuntimeCache; } if ($this->isBuilding) { throw new \RuntimeException(sprintf('Content repository "%s" was attempted to be build in recursion.', $this->contentRepositoryId->value), 1730552199); } $this->isBuilding = true; - $contentGraphReadModel = $this->projectionsAndCatchUpHooks->contentGraphProjection->getState(); + $contentGraphReadModel = $this->contentGraphProjection->getState(); $commandHandlingDependencies = new CommandHandlingDependencies($contentGraphReadModel); // we dont need full recursion in rebase - e.g apply workspace commands - and thus we can use this set for simulation $commandBusForRebaseableCommands = new CommandBus( $commandHandlingDependencies, new NodeAggregateCommandHandler( - $this->projectionFactoryDependencies->nodeTypeManager, - $this->projectionFactoryDependencies->contentDimensionZookeeper, - $this->projectionFactoryDependencies->interDimensionalVariationGraph, - $this->projectionFactoryDependencies->propertyConverter, + $this->nodeTypeManager, + $this->contentDimensionZookeeper, + $this->interDimensionalVariationGraph, + $this->propertyConverter, ), new DimensionSpaceCommandHandler( - $this->projectionFactoryDependencies->contentDimensionZookeeper, - $this->projectionFactoryDependencies->interDimensionalVariationGraph, + $this->contentDimensionZookeeper, + $this->interDimensionalVariationGraph, ), new NodeDuplicationCommandHandler( - $this->projectionFactoryDependencies->nodeTypeManager, - $this->projectionFactoryDependencies->contentDimensionZookeeper, - $this->projectionFactoryDependencies->interDimensionalVariationGraph, + $this->nodeTypeManager, + $this->contentDimensionZookeeper, + $this->interDimensionalVariationGraph, ) ); $commandSimulatorFactory = new CommandSimulatorFactory( - $this->projectionsAndCatchUpHooks->contentGraphProjection, - $this->projectionFactoryDependencies->eventNormalizer, + $this->contentGraphProjection, + $this->eventNormalizer, $commandBusForRebaseableCommands ); $publicCommandBus = $commandBusForRebaseableCommands->withAdditionalHandlers( new WorkspaceCommandHandler( $commandSimulatorFactory, - $this->projectionFactoryDependencies->eventStore, - $this->projectionFactoryDependencies->eventNormalizer, + $this->eventStore, + $this->eventNormalizer, ) ); $authProvider = $this->authProviderFactory->build($this->contentRepositoryId, $contentGraphReadModel); $commandHooks = $this->commandHooksFactory->build(CommandHooksFactoryDependencies::create( $this->contentRepositoryId, - $this->projectionsAndCatchUpHooks->contentGraphProjection->getState(), - $this->projectionFactoryDependencies->nodeTypeManager, - $this->projectionFactoryDependencies->contentDimensionSource, - $this->projectionFactoryDependencies->interDimensionalVariationGraph, + $this->contentGraphProjection->getState(), + $this->nodeTypeManager, + $this->contentDimensionSource, + $this->interDimensionalVariationGraph, )); - $this->contentRepository = new ContentRepository( + $this->contentRepositoryRuntimeCache = new ContentRepository( $this->contentRepositoryId, $publicCommandBus, - $this->projectionFactoryDependencies->eventStore, - $this->projectionsAndCatchUpHooks, - $this->projectionFactoryDependencies->eventNormalizer, - $this->buildEventPersister(), - $this->projectionFactoryDependencies->nodeTypeManager, - $this->projectionFactoryDependencies->interDimensionalVariationGraph, - $this->projectionFactoryDependencies->contentDimensionSource, + $this->eventStore, + $this->eventNormalizer, + $this->subscriptionEngine, + $this->nodeTypeManager, + $this->interDimensionalVariationGraph, + $this->contentDimensionSource, $authProvider, $this->clock, $contentGraphReadModel, $commandHooks, + $this->additionalProjectionStates, ); $this->isBuilding = false; - return $this->contentRepository; + return $this->contentRepositoryRuntimeCache; } /** @@ -175,24 +220,19 @@ public function getOrBuild(): ContentRepository public function buildService( ContentRepositoryServiceFactoryInterface $serviceFactory ): ContentRepositoryServiceInterface { - $serviceFactoryDependencies = ContentRepositoryServiceFactoryDependencies::create( - $this->projectionFactoryDependencies, + $this->contentRepositoryId, + $this->eventStore, + $this->eventNormalizer, + $this->nodeTypeManager, + $this->contentDimensionSource, + $this->contentDimensionZookeeper, + $this->interDimensionalVariationGraph, + $this->propertyConverter, $this->getOrBuild(), - $this->buildEventPersister(), - $this->projectionsAndCatchUpHooks, + $this->contentGraphProjection->getState(), + $this->subscriptionEngine, ); return $serviceFactory->build($serviceFactoryDependencies); } - - private function buildEventPersister(): EventPersister - { - if (!$this->eventPersister) { - $this->eventPersister = new EventPersister( - $this->projectionFactoryDependencies->eventStore, - $this->projectionFactoryDependencies->eventNormalizer, - ); - } - return $this->eventPersister; - } } diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php index 08ea272181d..4bc420a7986 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositoryServiceFactoryDependencies.php @@ -19,11 +19,11 @@ use Neos\ContentRepository\Core\DimensionSpace\ContentDimensionZookeeper; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; use Neos\ContentRepository\Core\EventStore\EventNormalizer; -use Neos\ContentRepository\Core\EventStore\EventPersister; use Neos\ContentRepository\Core\Infrastructure\Property\PropertyConverter; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; -use Neos\ContentRepository\Core\Projection\ProjectionsAndCatchUpHooks; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\EventStore\EventStoreInterface; /** @@ -34,7 +34,6 @@ final readonly class ContentRepositoryServiceFactoryDependencies { private function __construct( - // These properties are from ProjectionFactoryDependencies public ContentRepositoryId $contentRepositoryId, public EventStoreInterface $eventStore, public EventNormalizer $eventNormalizer, @@ -44,9 +43,8 @@ private function __construct( public InterDimensionalVariationGraph $interDimensionalVariationGraph, public PropertyConverter $propertyConverter, public ContentRepository $contentRepository, - // we don't need CommandBus, because this is included in ContentRepository->handle() - public EventPersister $eventPersister, - public ProjectionsAndCatchUpHooks $projectionsAndCatchUpHooks, + public ContentGraphReadModelInterface $contentGraphReadModel, + public SubscriptionEngine $subscriptionEngine, ) { } @@ -54,23 +52,30 @@ private function __construct( * @internal */ public static function create( - ProjectionFactoryDependencies $projectionFactoryDependencies, + ContentRepositoryId $contentRepositoryId, + EventStoreInterface $eventStore, + EventNormalizer $eventNormalizer, + NodeTypeManager $nodeTypeManager, + ContentDimensionSourceInterface $contentDimensionSource, + ContentDimensionZookeeper $contentDimensionZookeeper, + InterDimensionalVariationGraph $interDimensionalVariationGraph, + PropertyConverter $propertyConverter, ContentRepository $contentRepository, - EventPersister $eventPersister, - ProjectionsAndCatchUpHooks $projectionsAndCatchUpHooks, + ContentGraphReadModelInterface $contentGraphReadModel, + SubscriptionEngine $subscriptionEngine, ): self { return new self( - $projectionFactoryDependencies->contentRepositoryId, - $projectionFactoryDependencies->eventStore, - $projectionFactoryDependencies->eventNormalizer, - $projectionFactoryDependencies->nodeTypeManager, - $projectionFactoryDependencies->contentDimensionSource, - $projectionFactoryDependencies->contentDimensionZookeeper, - $projectionFactoryDependencies->interDimensionalVariationGraph, - $projectionFactoryDependencies->propertyConverter, + $contentRepositoryId, + $eventStore, + $eventNormalizer, + $nodeTypeManager, + $contentDimensionSource, + $contentDimensionZookeeper, + $interDimensionalVariationGraph, + $propertyConverter, $contentRepository, - $eventPersister, - $projectionsAndCatchUpHooks, + $contentGraphReadModel, + $subscriptionEngine, ); } } diff --git a/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php new file mode 100644 index 00000000000..399573c23a8 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Factory/ContentRepositorySubscriberFactories.php @@ -0,0 +1,46 @@ + + * @internal only API for custom content repository integrations + */ +final class ContentRepositorySubscriberFactories implements \IteratorAggregate +{ + /** + * @var array + */ + private array $subscriberFactories; + + private function __construct(ProjectionSubscriberFactory ...$subscriberFactories) + { + $this->subscriberFactories = $subscriberFactories; + } + + /** + * @param array $subscriberFactories + * @return self + */ + public static function fromArray(array $subscriberFactories): self + { + return new self(...$subscriberFactories); + } + + public static function none(): self + { + return new self(); + } + + public function isEmpty(): bool + { + return $this->subscriberFactories === []; + } + + public function getIterator(): \Traversable + { + yield from $this->subscriberFactories; + } +} diff --git a/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php new file mode 100644 index 00000000000..203936237e3 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Factory/ProjectionSubscriberFactory.php @@ -0,0 +1,60 @@ +> $projectionFactory + * @param CatchUpHookFactoryInterface|null $catchUpHookFactory + * @param array $projectionFactoryOptions + */ + public function __construct( + private SubscriptionId $subscriptionId, + private ProjectionFactoryInterface $projectionFactory, + private ?CatchUpHookFactoryInterface $catchUpHookFactory, + private array $projectionFactoryOptions, + ) { + } + + public function build(SubscriberFactoryDependencies $dependencies): ProjectionSubscriber + { + $projection = $this->projectionFactory->build($dependencies, $this->projectionFactoryOptions); + $catchUpHook = $this->catchUpHookFactory?->build(CatchUpHookFactoryDependencies::create( + $dependencies->contentRepositoryId, + $projection->getState(), + $dependencies->nodeTypeManager, + $dependencies->contentDimensionSource, + $dependencies->interDimensionalVariationGraph, + )); + + return new ProjectionSubscriber( + $this->subscriptionId, + $projection, + $catchUpHook, + ); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Factory/ProjectionsAndCatchUpHooksFactory.php b/Neos.ContentRepository.Core/Classes/Factory/ProjectionsAndCatchUpHooksFactory.php deleted file mode 100644 index f48ed1345c0..00000000000 --- a/Neos.ContentRepository.Core/Classes/Factory/ProjectionsAndCatchUpHooksFactory.php +++ /dev/null @@ -1,92 +0,0 @@ ->, options: array, catchUpHooksFactories: array>}> - */ - private array $factories = []; - - /** - * @param ProjectionFactoryInterface> $factory - * @param array $options - * @return void - * @api - */ - public function registerFactory(ProjectionFactoryInterface $factory, array $options): void - { - $this->factories[get_class($factory)] = [ - 'factory' => $factory, - 'options' => $options, - 'catchUpHooksFactories' => [] - ]; - } - - /** - * @param ProjectionFactoryInterface> $factory - * @param CatchUpHookFactoryInterface $catchUpHookFactory - * @return void - * @api - */ - public function registerCatchUpHookFactory(ProjectionFactoryInterface $factory, CatchUpHookFactoryInterface $catchUpHookFactory): void - { - $this->factories[get_class($factory)]['catchUpHooksFactories'][] = $catchUpHookFactory; - } - - /** - * @internal this method is only called by the {@see ContentRepositoryFactory}, and not by anybody in userland - */ - public function build(ProjectionFactoryDependencies $projectionFactoryDependencies): ProjectionsAndCatchUpHooks - { - $contentGraphProjection = null; - $projectionsArray = []; - $catchUpHookFactoriesByProjectionClassName = []; - foreach ($this->factories as $factoryDefinition) { - $factory = $factoryDefinition['factory']; - $options = $factoryDefinition['options']; - assert($factory instanceof ProjectionFactoryInterface); - - $catchUpHookFactories = CatchUpHookFactories::create(); - foreach ($factoryDefinition['catchUpHooksFactories'] as $catchUpHookFactory) { - assert($catchUpHookFactory instanceof CatchUpHookFactoryInterface); - $catchUpHookFactories = $catchUpHookFactories->with($catchUpHookFactory); - } - - $projection = $factory->build( - $projectionFactoryDependencies, - $options, - ); - $catchUpHookFactoriesByProjectionClassName[$projection::class] = $catchUpHookFactories; - if ($projection instanceof ContentGraphProjectionInterface) { - if ($contentGraphProjection !== null) { - throw new \RuntimeException(sprintf('Content repository requires exactly one %s to be registered.', ContentGraphProjectionInterface::class)); - } - $contentGraphProjection = $projection; - } else { - $projectionsArray[] = $projection; - } - } - - if ($contentGraphProjection === null) { - throw new \RuntimeException(sprintf('Content repository requires the %s to be registered.', ContentGraphProjectionInterface::class)); - } - - return new ProjectionsAndCatchUpHooks($contentGraphProjection, Projections::fromArray($projectionsArray), $catchUpHookFactoriesByProjectionClassName); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Factory/ProjectionFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php similarity index 53% rename from Neos.ContentRepository.Core/Classes/Factory/ProjectionFactoryDependencies.php rename to Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php index 9bb2f0cc31f..1bd43162bc4 100644 --- a/Neos.ContentRepository.Core/Classes/Factory/ProjectionFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Factory/SubscriberFactoryDependencies.php @@ -15,28 +15,49 @@ namespace Neos\ContentRepository\Core\Factory; use Neos\ContentRepository\Core\Dimension\ContentDimensionSourceInterface; -use Neos\ContentRepository\Core\DimensionSpace\ContentDimensionZookeeper; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; -use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\Infrastructure\Property\PropertyConverter; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; -use Neos\EventStore\EventStoreInterface; /** * @api because it is used inside the ProjectionsFactory */ -final readonly class ProjectionFactoryDependencies +final readonly class SubscriberFactoryDependencies { - public function __construct( + private function __construct( public ContentRepositoryId $contentRepositoryId, - public EventStoreInterface $eventStore, - public EventNormalizer $eventNormalizer, public NodeTypeManager $nodeTypeManager, public ContentDimensionSourceInterface $contentDimensionSource, - public ContentDimensionZookeeper $contentDimensionZookeeper, public InterDimensionalVariationGraph $interDimensionalVariationGraph, - public PropertyConverter $propertyConverter, + private PropertyConverter $propertyConverter, ) { } + + /** + * @internal + */ + public static function create( + ContentRepositoryId $contentRepositoryId, + NodeTypeManager $nodeTypeManager, + ContentDimensionSourceInterface $contentDimensionSource, + InterDimensionalVariationGraph $interDimensionalVariationGraph, + PropertyConverter $propertyConverter + ): self { + return new self( + $contentRepositoryId, + $nodeTypeManager, + $contentDimensionSource, + $interDimensionalVariationGraph, + $propertyConverter + ); + } + + /** + * @internal only to be used for custom content graph integrations to build a node property collection + */ + public function getPropertyConverter(): PropertyConverter + { + return $this->propertyConverter; + } } diff --git a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamHandling.php b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamHandling.php index a228ca7c864..a16aff28aa0 100644 --- a/Neos.ContentRepository.Core/Classes/Feature/ContentStreamHandling.php +++ b/Neos.ContentRepository.Core/Classes/Feature/ContentStreamHandling.php @@ -18,6 +18,9 @@ use Neos\EventStore\Model\Event\Version; use Neos\EventStore\Model\EventStream\ExpectedVersion; +/** + * @internal + */ trait ContentStreamHandling { /** diff --git a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php b/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php deleted file mode 100644 index 8551d5febb8..00000000000 --- a/Neos.ContentRepository.Core/Classes/Infrastructure/DbalCheckpointStorage.php +++ /dev/null @@ -1,164 +0,0 @@ -connection->getDatabasePlatform(); - if (!($platform instanceof MySQLPlatform || $platform instanceof PostgreSqlPlatform)) { - throw new \InvalidArgumentException(sprintf('The %s only supports the platforms %s and %s currently. Given: %s', $this::class, MySQLPlatform::class, PostgreSQLPlatform::class, get_debug_type($platform)), 1660556004); - } - if (strlen($this->subscriberId) > 255) { - throw new \InvalidArgumentException('The subscriberId must not exceed 255 characters', 1705673456); - } - $this->platform = $platform; - } - - public function setUp(): void - { - foreach ($this->determineRequiredSqlStatements() as $statement) { - $this->connection->executeStatement($statement); - } - try { - $this->connection->insert($this->tableName, ['subscriberid' => $this->subscriberId, 'appliedsequencenumber' => 0]); - } catch (UniqueConstraintViolationException $e) { - // table and row already exists, ignore - } - } - - public function status(): CheckpointStorageStatus - { - try { - $this->connection->connect(); - } catch (\Throwable $e) { - return CheckpointStorageStatus::error(sprintf('Failed to connect to database for subscriber "%s": %s', $this->subscriberId, $e->getMessage())); - } - try { - $requiredSqlStatements = $this->determineRequiredSqlStatements(); - } catch (\Throwable $e) { - return CheckpointStorageStatus::error(sprintf('Failed to compare database schema for subscriber "%s": %s', $this->subscriberId, $e->getMessage())); - } - if ($requiredSqlStatements !== []) { - return CheckpointStorageStatus::setupRequired(sprintf('The following SQL statement%s required for subscriber "%s": %s', count($requiredSqlStatements) !== 1 ? 's are' : ' is', $this->subscriberId, implode(chr(10), $requiredSqlStatements))); - } - try { - $appliedSequenceNumber = $this->connection->fetchOne('SELECT appliedsequencenumber FROM ' . $this->tableName . ' WHERE subscriberid = :subscriberId', ['subscriberId' => $this->subscriberId]); - } catch (\Throwable $e) { - return CheckpointStorageStatus::error(sprintf('Failed to determine initial applied sequence number for subscriber "%s": %s', $this->subscriberId, $e->getMessage())); - } - if ($appliedSequenceNumber === false) { - return CheckpointStorageStatus::setupRequired(sprintf('Initial initial applied sequence number not set for subscriber "%s"', $this->subscriberId)); - } - return CheckpointStorageStatus::ok(); - } - - public function acquireLock(): SequenceNumber - { - if ($this->connection->isTransactionActive()) { - throw new \RuntimeException(sprintf('Failed to acquire checkpoint lock for subscriber "%s" because a transaction is active already', $this->subscriberId), 1652268416); - } - $this->connection->beginTransaction(); - try { - $highestAppliedSequenceNumber = $this->connection->fetchOne('SELECT appliedsequencenumber FROM ' . $this->connection->quoteIdentifier($this->tableName) . ' WHERE subscriberid = :subscriberId ' . $this->platform->getForUpdateSQL() . ' NOWAIT', [ - 'subscriberId' => $this->subscriberId - ]); - } catch (DBALException $exception) { - $this->connection->rollBack(); - if ($exception instanceof LockWaitTimeoutException || ($exception instanceof DBALDriverException && ($exception->getCode() === 3572 || $exception->getCode() === 7))) { - throw new \RuntimeException(sprintf('Failed to acquire checkpoint lock for subscriber "%s" because it is acquired already', $this->subscriberId), 1652279016); - } - throw new \RuntimeException($exception->getMessage(), 1544207778, $exception); - } - if (!is_numeric($highestAppliedSequenceNumber)) { - $this->connection->rollBack(); - throw new \RuntimeException(sprintf('Failed to fetch highest applied sequence number for subscriber "%s". Please run %s::setUp()', $this->subscriberId, $this::class), 1652279139); - } - $this->lockedSequenceNumber = SequenceNumber::fromInteger((int)$highestAppliedSequenceNumber); - return $this->lockedSequenceNumber; - } - - public function updateAndReleaseLock(SequenceNumber $sequenceNumber): void - { - if ($this->lockedSequenceNumber === null) { - throw new \RuntimeException(sprintf('Failed to update and commit checkpoint for subscriber "%s" because the lock has not been acquired successfully before', $this->subscriberId), 1660556344); - } - if (!$this->connection->isTransactionActive()) { - throw new \RuntimeException(sprintf('Failed to update and commit checkpoint for subscriber "%s" because no transaction is active', $this->subscriberId), 1652279314); - } - if ($this->connection->isRollbackOnly()) { - // TODO as described in https://github.com/neos/neos-development-collection/issues/4970 we are in a bad state and cannot commit after a nested transaction was rolled back. - throw new \RuntimeException(sprintf('Failed to update and commit checkpoint for subscriber "%s" because the transaction has been marked for rollback only. See https://github.com/neos/neos-development-collection/issues/4970', $this->subscriberId), 1711964313); - } - try { - if (!$this->lockedSequenceNumber->equals($sequenceNumber)) { - $this->connection->update($this->tableName, ['appliedsequencenumber' => $sequenceNumber->value], ['subscriberid' => $this->subscriberId]); - } - $this->connection->commit(); - } catch (DBALException $exception) { - $this->connection->rollBack(); - throw new \RuntimeException(sprintf('Failed to update and commit highest applied sequence number for subscriber "%s". Please run %s::setUp()', $this->subscriberId, $this::class), 1652279375, $exception); - } finally { - $this->lockedSequenceNumber = null; - } - } - - public function getHighestAppliedSequenceNumber(): SequenceNumber - { - $highestAppliedSequenceNumber = $this->connection->fetchOne('SELECT appliedsequencenumber FROM ' . $this->connection->quoteIdentifier($this->tableName) . ' WHERE subscriberid = :subscriberId ', [ - 'subscriberId' => $this->subscriberId - ]); - if (!is_numeric($highestAppliedSequenceNumber)) { - throw new \RuntimeException(sprintf('Failed to fetch highest applied sequence number for subscriber "%s". Please run %s::setUp()', $this->subscriberId, $this::class), 1652279427); - } - return SequenceNumber::fromInteger((int)$highestAppliedSequenceNumber); - } - - // -------------- - - /** - * @return array - */ - private function determineRequiredSqlStatements(): array - { - $schemaManager = $this->connection->createSchemaManager(); - $tableSchema = new Table( - $this->tableName, - [ - (new Column('subscriberid', Type::getType(Types::STRING)))->setLength(255), - (new Column('appliedsequencenumber', Type::getType(Types::INTEGER))) - ] - ); - $tableSchema->setPrimaryKey(['subscriberid']); - $schema = DbalSchemaFactory::createSchemaWithTables($schemaManager, [$tableSchema]); - return DbalSchemaDiff::determineRequiredSqlStatements($this->connection, $schema); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUp.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUp.php deleted file mode 100644 index 35cd26467a9..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUp.php +++ /dev/null @@ -1,134 +0,0 @@ -batchSize < 1) { - throw new \InvalidArgumentException(sprintf('batch size must be a positive integer, given: %d', $this->batchSize), 1705672467); - } - } - - /** - * @param \Closure(EventEnvelope): void $eventHandler The callback that is invoked for every {@see EventEnvelope} that is processed - * @param CheckpointStorageInterface $checkpointStorage The checkpoint storage that saves the last processed {@see SequenceNumber} - */ - public static function create(\Closure $eventHandler, CheckpointStorageInterface $checkpointStorage): self - { - return new self($eventHandler, $checkpointStorage, 1, null); - } - - /** - * After how many events should the (database) transaction be committed? - * - * @param int $batchSize Number of events to process before the checkpoint is written - */ - public function withBatchSize(int $batchSize): self - { - if ($batchSize === $this->batchSize) { - return $this; - } - return new self($this->eventHandler, $this->checkpointStorage, $batchSize, $this->onBeforeBatchCompletedHook); - } - - /** - * This hook is called directly before the sequence number is persisted back in CheckpointStorage. - * Use this to trigger any operation which need to happen BEFORE the sequence number update is made - * visible to the outside. - * - * Overrides all previously registered onBeforeBatchCompleted hooks. - * - * @param \Closure(): void $callback the hook being called before the batch is completed - */ - public function withOnBeforeBatchCompleted(\Closure $callback): self - { - return new self($this->eventHandler, $this->checkpointStorage, $this->batchSize, $callback); - } - - /** - * Iterate over the $eventStream, invoke the specified event handler closure for every {@see EventEnvelope} and update - * the last processed sequence number in the {@see CheckpointStorageInterface} - * - * @param EventStreamInterface $eventStream The event stream to process - * @return SequenceNumber The last processed {@see SequenceNumber} - * @throws \Throwable Exceptions that are thrown during callback handling are re-thrown - */ - public function run(EventStreamInterface $eventStream): SequenceNumber - { - $highestAppliedSequenceNumber = $this->checkpointStorage->acquireLock(); - $iteration = 0; - try { - foreach ($eventStream->withMinimumSequenceNumber($highestAppliedSequenceNumber->next()) as $eventEnvelope) { - if ($eventEnvelope->sequenceNumber->value <= $highestAppliedSequenceNumber->value) { - continue; - } - try { - ($this->eventHandler)($eventEnvelope); - } catch (\Exception $e) { - throw new \RuntimeException(sprintf('Exception while catching up to sequence number %d: %s', $eventEnvelope->sequenceNumber->value, $e->getMessage()), 1710707311, $e); - } - $iteration++; - if ($this->batchSize === 1 || $iteration % $this->batchSize === 0) { - if ($this->onBeforeBatchCompletedHook) { - ($this->onBeforeBatchCompletedHook)(); - } - $this->checkpointStorage->updateAndReleaseLock($eventEnvelope->sequenceNumber); - $highestAppliedSequenceNumber = $this->checkpointStorage->acquireLock(); - } else { - $highestAppliedSequenceNumber = $eventEnvelope->sequenceNumber; - } - } - } finally { - try { - if ($this->onBeforeBatchCompletedHook) { - ($this->onBeforeBatchCompletedHook)(); - } - } finally { - $this->checkpointStorage->updateAndReleaseLock($highestAppliedSequenceNumber); - } - } - return $highestAppliedSequenceNumber; - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactories.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php similarity index 88% rename from Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactories.php rename to Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php index efa364124ba..22f2621e2a2 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactories.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactories.php @@ -2,11 +2,13 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Core\Projection; +namespace Neos\ContentRepository\Core\Projection\CatchUpHook; + +use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; /** * @implements CatchUpHookFactoryInterface - * @internal + * @api */ final class CatchUpHookFactories implements CatchUpHookFactoryInterface { @@ -30,7 +32,6 @@ public static function create(): self /** * @param CatchUpHookFactoryInterface $catchUpHookFactory - * @return self */ public function with(CatchUpHookFactoryInterface $catchUpHookFactory): self { @@ -50,6 +51,11 @@ private function has(string $catchUpHookFactoryClassName): bool return array_key_exists($catchUpHookFactoryClassName, $this->catchUpHookFactories); } + public function isEmpty(): bool + { + return $this->catchUpHookFactories === []; + } + public function build(CatchUpHookFactoryDependencies $dependencies): CatchUpHookInterface { $catchUpHooks = array_map(static fn(CatchUpHookFactoryInterface $catchUpHookFactory) => $catchUpHookFactory->build($dependencies), $this->catchUpHookFactories); diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryDependencies.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryDependencies.php similarity index 94% rename from Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryDependencies.php rename to Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryDependencies.php index 0a98f7e11c3..2b891f34336 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryDependencies.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryDependencies.php @@ -12,11 +12,12 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Core\Projection; +namespace Neos\ContentRepository\Core\Projection\CatchUpHook; use Neos\ContentRepository\Core\Dimension\ContentDimensionSourceInterface; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; +use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; /** diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryInterface.php similarity index 83% rename from Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryInterface.php rename to Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryInterface.php index 82fc7fea7b4..86e06d571cc 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookFactoryInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFactoryInterface.php @@ -2,7 +2,9 @@ declare(strict_types=1); -namespace Neos\ContentRepository\Core\Projection; +namespace Neos\ContentRepository\Core\Projection\CatchUpHook; + +use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; /** * @template T of ProjectionStateInterface diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFailed.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFailed.php new file mode 100644 index 00000000000..487bbd9e40e --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHook/CatchUpHookFailed.php @@ -0,0 +1,14 @@ +catchUpHooks = $catchUpHooks; + } + + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void + { + $this->delegateHooks( + fn (CatchUpHookInterface $catchUpHook) => $catchUpHook->onBeforeCatchUp($subscriptionStatus), + 'onBeforeCatchUp' + ); + } + + public function onBeforeEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void + { + $this->delegateHooks( + fn (CatchUpHookInterface $catchUpHook) => $catchUpHook->onBeforeEvent($eventInstance, $eventEnvelope), + 'onBeforeEvent' + ); + } + + public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void + { + $this->delegateHooks( + fn (CatchUpHookInterface $catchUpHook) => $catchUpHook->onAfterEvent($eventInstance, $eventEnvelope), + 'onAfterEvent' + ); + } + + public function onAfterBatchCompleted(): void + { + $this->delegateHooks( + fn (CatchUpHookInterface $catchUpHook) => $catchUpHook->onAfterBatchCompleted(), + 'onAfterBatchCompleted' + ); + } + + public function onAfterCatchUp(): void + { + $this->delegateHooks( + fn (CatchUpHookInterface $catchUpHook) => $catchUpHook->onAfterCatchUp(), + 'onAfterCatchUp' + ); + } + + /** + * @param \Closure(CatchUpHookInterface): void $closure + * @return void + */ + private function delegateHooks(\Closure $closure, string $hookName): void + { + $firstError = null; + /** @var list $errors */ + $errors = []; + foreach ($this->catchUpHooks as $catchUpHook) { + try { + $closure($catchUpHook); + } catch (\Throwable $e) { + $firstError ??= $e; + $failedCatchupHookName = substr(strrchr($catchUpHook::class, '\\') ?: '', 1); + $errors[] = sprintf('"%s": %s', $failedCatchupHookName, $e->getMessage()); + } + } + if ($firstError !== null) { + throw new CatchUpHookFailed( + sprintf('Hook "%s" failed: %s', $hookName, join(";\n", $errors)), + 1733243960, + $firstError + ); + } + } +} diff --git a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookInterface.php deleted file mode 100644 index fbdfed6e8d5..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/CatchUpHookInterface.php +++ /dev/null @@ -1,59 +0,0 @@ -maximumSequenceNumber, - $progressCallback ?? $this->progressCallback, - ); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageInterface.php b/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageInterface.php deleted file mode 100644 index 43dc37f8ab7..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageInterface.php +++ /dev/null @@ -1,61 +0,0 @@ -type, $details); - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatusType.php b/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatusType.php deleted file mode 100644 index 3e138d4348d..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/CheckpointStorageStatusType.php +++ /dev/null @@ -1,12 +0,0 @@ - * @api for creating a custom content repository graph projection implementation, **not for users of the CR** */ -interface ContentGraphProjectionFactoryInterface extends ProjectionFactoryInterface +interface ContentGraphProjectionFactoryInterface { - /** - * @param array $options - */ public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, - array $options, + SubscriberFactoryDependencies $projectionFactoryDependencies, ): ContentGraphProjectionInterface; } diff --git a/Neos.ContentRepository.Core/Classes/Projection/DelegatingCatchUpHook.php b/Neos.ContentRepository.Core/Classes/Projection/DelegatingCatchUpHook.php deleted file mode 100644 index 3edc73b751d..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/DelegatingCatchUpHook.php +++ /dev/null @@ -1,63 +0,0 @@ -catchUpHooks = $catchUpHooks; - } - - public function onBeforeCatchUp(): void - { - foreach ($this->catchUpHooks as $catchUpHook) { - $catchUpHook->onBeforeCatchUp(); - } - } - - public function onBeforeEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void - { - foreach ($this->catchUpHooks as $catchUpHook) { - $catchUpHook->onBeforeEvent($eventInstance, $eventEnvelope); - } - } - - public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $eventEnvelope): void - { - foreach ($this->catchUpHooks as $catchUpHook) { - $catchUpHook->onAfterEvent($eventInstance, $eventEnvelope); - } - } - - public function onBeforeBatchCompleted(): void - { - foreach ($this->catchUpHooks as $catchUpHook) { - $catchUpHook->onBeforeBatchCompleted(); - } - } - - public function onAfterCatchUp(): void - { - foreach ($this->catchUpHooks as $catchUpHook) { - $catchUpHook->onAfterCatchUp(); - } - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionFactoryInterface.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionFactoryInterface.php index 6c0de0992d5..d2f03269b1d 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionFactoryInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionFactoryInterface.php @@ -4,7 +4,7 @@ namespace Neos\ContentRepository\Core\Projection; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; /** * @template-covariant T of ProjectionInterface @@ -17,7 +17,7 @@ interface ProjectionFactoryInterface * @return T */ public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, + SubscriberFactoryDependencies $projectionFactoryDependencies, array $options, ): ProjectionInterface; } diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php index 22c65ced9f7..aff57389895 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionInterface.php @@ -21,21 +21,17 @@ interface ProjectionInterface { /** - * Set up the projection state (create databases, call {@see CheckpointStorageInterface::setUp()}). + * Set up the projection state (create/update required database tables, ...). */ public function setUp(): void; /** - * Determines the status of the projection (not to confuse with {@see getState()}) + * Determines the setup status of the projection. E.g. are the database tables created or any columns missing. */ public function status(): ProjectionStatus; - public function canHandle(EventInterface $event): bool; - public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void; - public function getCheckpointStorage(): CheckpointStorageInterface; - /** * NOTE: The ProjectionStateInterface returned must be ALWAYS THE SAME INSTANCE. * @@ -46,5 +42,5 @@ public function getCheckpointStorage(): CheckpointStorageInterface; */ public function getState(): ProjectionStateInterface; - public function reset(): void; + public function resetState(): void; } diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php new file mode 100644 index 00000000000..81b6d82a0eb --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStates.php @@ -0,0 +1,70 @@ +, ProjectionStateInterface> $statesByClassName + */ + private function __construct( + private array $statesByClassName, + ) { + } + + public static function createEmpty(): self + { + return new self([]); + } + + /** + * @param array $states + */ + public static function fromArray(array $states): self + { + $statesByClassName = []; + foreach ($states as $state) { + if (!$state instanceof ProjectionStateInterface) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', ProjectionStateInterface::class, get_debug_type($state)), 1729687661); + } + if ($state instanceof ContentGraphReadModelInterface) { + throw new \InvalidArgumentException(sprintf('The content graph state (%s) must not be part of the additional projection states', ContentGraphReadModelInterface::class), 1732390657); + } + if (array_key_exists($state::class, $statesByClassName)) { + throw new \InvalidArgumentException(sprintf('An instance of %s is already part of the set', $state::class), 1729687716); + } + $statesByClassName[$state::class] = $state; + } + return new self($statesByClassName); + } + + /** + * Retrieve a single state (aka read model) by its fully qualified PHP class name + * + * @template T of ProjectionStateInterface + * @param class-string $className + * @return T + * @throws \InvalidArgumentException if the specified state class is not registered + */ + public function get(string $className): ProjectionStateInterface + { + if ($className === ContentGraphReadModelInterface::class) { + throw new \InvalidArgumentException(sprintf('Accessing the content repository projection state (%s) via is not allowed. Please use the API on the content repository instead.', ContentGraphReadModelInterface::class), 1732390824); + } + if (!array_key_exists($className, $this->statesByClassName)) { + throw new \InvalidArgumentException(sprintf('A projection state of type "%s" is not registered in this content repository.', $className), 1662033650); + } + /** @var T $state */ + $state = $this->statesByClassName[$className]; + return $state; + } +} diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatus.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatus.php index 517c53d7a7c..17ddc848176 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatus.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatus.php @@ -5,6 +5,10 @@ namespace Neos\ContentRepository\Core\Projection; /** + * The setup status of a projection. + * + * E.g. are the database tables created or any columns missing. + * * @api */ final readonly class ProjectionStatus @@ -35,20 +39,4 @@ public static function setupRequired(string $details): self { return new self(ProjectionStatusType::SETUP_REQUIRED, $details); } - - /** - * @param non-empty-string $details - */ - public static function replayRequired(string $details): self - { - return new self(ProjectionStatusType::REPLAY_REQUIRED, $details); - } - - /** - * @param non-empty-string $details - */ - public function withDetails(string $details): self - { - return new self($this->type, $details); - } } diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatusType.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatusType.php index 48681927742..285ad8e86d4 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatusType.php +++ b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatusType.php @@ -9,8 +9,17 @@ */ enum ProjectionStatusType { + /** + * No actions needed + */ case OK; - case ERROR; + /** + * The projection needs to be setup to adjust its schema + * {@see \Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer::setUp()} + */ case SETUP_REQUIRED; - case REPLAY_REQUIRED; + /** + * An error occurred while determining the status (e.g. connection is closed) + */ + case ERROR; } diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatuses.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatuses.php deleted file mode 100644 index cc146b1a76e..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionStatuses.php +++ /dev/null @@ -1,41 +0,0 @@ - - */ -final readonly class ProjectionStatuses implements \IteratorAggregate -{ - /** - * @param array>, ProjectionStatus> $statuses - */ - private function __construct( - public array $statuses, - ) { - } - - public static function createEmpty(): self - { - return new self([]); - } - - /** - * @param class-string> $projectionClassName - */ - public function with(string $projectionClassName, ProjectionStatus $projectionStatus): self - { - $statuses = $this->statuses; - $statuses[$projectionClassName] = $projectionStatus; - return new self($statuses); - } - - - public function getIterator(): \Traversable - { - yield from $this->statuses; - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/Projections.php b/Neos.ContentRepository.Core/Classes/Projection/Projections.php index e8fd3995a32..a66d0ee9e3b 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/Projections.php +++ b/Neos.ContentRepository.Core/Classes/Projection/Projections.php @@ -4,6 +4,8 @@ namespace Neos\ContentRepository\Core\Projection; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; + /** * An immutable set of Content Repository projections ({@see ProjectionInterface} * @@ -13,7 +15,7 @@ final class Projections implements \IteratorAggregate, \Countable { /** - * @var array>, ProjectionInterface> + * @var array> */ private array $projections; @@ -32,7 +34,7 @@ public static function empty(): self } /** - * @param array> $projections + * @param array> $projections * @return self */ public static function fromArray(array $projections): self @@ -54,45 +56,28 @@ public static function fromArray(array $projections): self } /** - * @template T of ProjectionInterface - * @param class-string $projectionClassName - * @return T + * @return ProjectionInterface */ - public function get(string $projectionClassName): ProjectionInterface + public function get(SubscriptionId $id): ProjectionInterface { - $projection = $this->projections[$projectionClassName] ?? null; - if (!$projection instanceof $projectionClassName) { - throw new \InvalidArgumentException( - sprintf( - 'a projection of type "%s" is not registered in this content repository instance.', - $projectionClassName - ), - 1650120813 - ); + if (!$this->has($id)) { + throw new \InvalidArgumentException(sprintf('a projection with id "%s" is not registered in this content repository instance.', $id->value), 1650120813); } - return $projection; + return $this->projections[$id->value]; } - public function has(string $projectionClassName): bool + public function has(SubscriptionId $id): bool { - return array_key_exists($projectionClassName, $this->projections); + return array_key_exists($id->value, $this->projections); } /** * @param ProjectionInterface $projection * @return self */ - public function with(ProjectionInterface $projection): self - { - return self::fromArray([...$this->projections, $projection]); - } - - /** - * @return list>> - */ - public function getClassNames(): array + public function with(SubscriptionId $id, ProjectionInterface $projection): self { - return array_keys($this->projections); + return self::fromArray([...$this->projections, ...[$id->value => $projection]]); } public function getIterator(): \Traversable diff --git a/Neos.ContentRepository.Core/Classes/Projection/ProjectionsAndCatchUpHooks.php b/Neos.ContentRepository.Core/Classes/Projection/ProjectionsAndCatchUpHooks.php deleted file mode 100644 index d7b674babdb..00000000000 --- a/Neos.ContentRepository.Core/Classes/Projection/ProjectionsAndCatchUpHooks.php +++ /dev/null @@ -1,35 +0,0 @@ ->, CatchUpHookFactories> $catchUpHookFactoriesByProjectionClassName - */ - public function __construct( - public ContentGraphProjectionInterface $contentGraphProjection, - Projections $additionalProjections, - private array $catchUpHookFactoriesByProjectionClassName, - ) { - $this->projections = $additionalProjections->with($this->contentGraphProjection); - } - - /** - * @param ProjectionInterface $projection - * @return ?CatchUpHookFactoryInterface - */ - public function getCatchUpHookFactoryForProjection(ProjectionInterface $projection): ?CatchUpHookFactoryInterface - { - return $this->catchUpHookFactoriesByProjectionClassName[$projection::class] ?? null; - } -} diff --git a/Neos.ContentRepository.Core/Classes/Projection/WithMarkStaleInterface.php b/Neos.ContentRepository.Core/Classes/Projection/WithMarkStaleInterface.php index baf93b2d1b5..30034de7358 100644 --- a/Neos.ContentRepository.Core/Classes/Projection/WithMarkStaleInterface.php +++ b/Neos.ContentRepository.Core/Classes/Projection/WithMarkStaleInterface.php @@ -4,8 +4,7 @@ namespace Neos\ContentRepository\Core\Projection; -use Neos\ContentRepository\Core\ContentRepository; -use Neos\ContentRepository\Core\EventStore\EventPersister; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; /** * Additional marker interface to add to a {@see ProjectionInterface}. @@ -19,7 +18,7 @@ interface WithMarkStaleInterface { /** * Triggered during catching up after applying events - * {@see ContentRepository::catchUpProjection()} + * {@see SubscriptionEngine::catchUpActive()} * * Can be f.e. used to flush caches inside the Projection State. * diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php new file mode 100644 index 00000000000..52bcf1edce5 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainer.php @@ -0,0 +1,228 @@ +eventStore->setup(); + $eventStoreIsEmpty = iterator_count($this->eventStore->load(VirtualStreamName::all())->limit(1)) === 0; + $setupResult = $this->subscriptionEngine->setup(); + if ($setupResult->errors !== null) { + return self::createErrorForReason('Setup failed:', $setupResult->errors); + } + if ($eventStoreIsEmpty) { + // note: possibly introduce $skipBooting flag instead + // see https://github.com/patchlevel/event-sourcing/blob/b8591c56b21b049f46bead8e7ab424fd2afe9917/src/Subscription/Engine/DefaultSubscriptionEngine.php#L42 + $bootResult = $this->subscriptionEngine->boot(); + if ($bootResult->errors !== null) { + return self::createErrorForReason('Initial catchup failed:', $bootResult->errors); + } + } + return null; + } + + public function status(): ContentRepositoryStatus + { + try { + $lastEventEnvelope = current(iterator_to_array($this->eventStore->load(VirtualStreamName::all())->backwards()->limit(1))) ?: null; + $sequenceNumber = $lastEventEnvelope?->sequenceNumber ?? SequenceNumber::none(); + } catch (DBALException) { + $sequenceNumber = null; + } + + return ContentRepositoryStatus::create( + $this->eventStore->status(), + $sequenceNumber, + $this->subscriptionEngine->subscriptionStatus() + ); + } + + public function replaySubscription(SubscriptionId $subscriptionId, \Closure|null $progressCallback = null): Error|null + { + $subscriptionStatus = $this->subscriptionEngine->subscriptionStatus(SubscriptionEngineCriteria::create([$subscriptionId]))->first(); + if ($subscriptionStatus === null) { + return new Error(sprintf('Subscription "%s" is not registered.', $subscriptionId->value)); + } + if ($subscriptionStatus->subscriptionStatus === SubscriptionStatus::NEW) { + return new Error(sprintf('Subscription "%s" is not setup and cannot be replayed.', $subscriptionId->value)); + } + $resetResult = $this->subscriptionEngine->reset(SubscriptionEngineCriteria::create([$subscriptionId])); + if ($resetResult->errors !== null) { + return self::createErrorForReason('Reset failed:', $resetResult->errors); + } + $bootResult = $this->subscriptionEngine->boot(SubscriptionEngineCriteria::create([$subscriptionId]), progressCallback: $progressCallback, batchSize: self::REPLAY_BATCH_SIZE); + if ($bootResult->errors !== null) { + return self::createErrorForReason('Catchup failed:', $bootResult->errors); + } + return null; + } + + public function replayAllSubscriptions(\Closure|null $progressCallback = null): Error|null + { + $resetResult = $this->subscriptionEngine->reset(); + if ($resetResult->errors !== null) { + return self::createErrorForReason('Reset failed:', $resetResult->errors); + } + $bootResult = $this->subscriptionEngine->boot(progressCallback: $progressCallback, batchSize: self::REPLAY_BATCH_SIZE); + if ($bootResult->errors !== null) { + return self::createErrorForReason('Catchup failed:', $bootResult->errors); + } + return null; + } + + /** + * WARNING: Removes all events from the content repository and resets the subscriptions + * This operation cannot be undone. + */ + public function prune(): Error|null + { + // prune all streams: + foreach ($this->findAllContentStreamStreamNames() as $contentStreamStreamName) { + $this->eventStore->deleteStream($contentStreamStreamName); + } + foreach ($this->findAllWorkspaceStreamNames() as $workspaceStreamName) { + $this->eventStore->deleteStream($workspaceStreamName); + } + $resetResult = $this->subscriptionEngine->reset(); + if ($resetResult->errors !== null) { + return self::createErrorForReason('Reset failed:', $resetResult->errors); + } + // note: possibly introduce $skipBooting flag like for setup + $bootResult = $this->subscriptionEngine->boot(); + if ($bootResult->errors !== null) { + return self::createErrorForReason('Catchup failed:', $bootResult->errors); + } + return null; + } + + private static function createErrorForReason(string $method, Errors $errors): Error + { + $message = [ + sprintf('%s Following error%s', $method, $errors->count() === 1 ? '' : 's'), + ...array_map(fn (string $line) => ' ' . $line, explode("\n", $errors->getClampedMessage())) + ]; + return new Error(join("\n", $message)); + } + + /** + * @return list + */ + private function findAllContentStreamStreamNames(): array + { + $events = $this->eventStore->load( + VirtualStreamName::forCategory(ContentStreamEventStreamName::EVENT_STREAM_NAME_PREFIX), + EventStreamFilter::create( + EventTypes::create( + // we are only interested in the creation events to limit the amount of events to fetch + EventType::fromString('ContentStreamWasCreated'), + EventType::fromString('ContentStreamWasForked') + ) + ) + ); + $allStreamNames = []; + foreach ($events as $eventEnvelope) { + $allStreamNames[] = $eventEnvelope->streamName; + } + return array_unique($allStreamNames, SORT_REGULAR); + } + + /** + * @return list + */ + private function findAllWorkspaceStreamNames(): array + { + $events = $this->eventStore->load( + VirtualStreamName::forCategory(WorkspaceEventStreamName::EVENT_STREAM_NAME_PREFIX), + EventStreamFilter::create( + EventTypes::create( + // we are only interested in the creation events to limit the amount of events to fetch + EventType::fromString('RootWorkspaceWasCreated'), + EventType::fromString('WorkspaceWasCreated') + ) + ) + ); + $allStreamNames = []; + foreach ($events as $eventEnvelope) { + $allStreamNames[] = $eventEnvelope->streamName; + } + return array_unique($allStreamNames, SORT_REGULAR); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainerFactory.php b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainerFactory.php new file mode 100644 index 00000000000..234b01fcb1d --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Service/ContentRepositoryMaintainerFactory.php @@ -0,0 +1,24 @@ + + * @api + */ +class ContentRepositoryMaintainerFactory implements ContentRepositoryServiceFactoryInterface +{ + public function build( + ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies + ): ContentRepositoryMaintainer { + return new ContentRepositoryMaintainer( + $serviceFactoryDependencies->eventStore, + $serviceFactoryDependencies->subscriptionEngine + ); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php index 6ed5d7c09fc..482586ab62e 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPruner.php @@ -4,7 +4,6 @@ namespace Neos\ContentRepository\Core\Service; -use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Feature\ContentStreamCreation\Event\ContentStreamWasCreated; @@ -21,10 +20,10 @@ use Neos\ContentRepository\Core\Service\ContentStreamPruner\ContentStreamForPruning; use Neos\ContentRepository\Core\Service\ContentStreamPruner\ContentStreamStatus; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event\EventType; use Neos\EventStore\Model\Event\EventTypes; -use Neos\EventStore\Model\Event\StreamName; use Neos\EventStore\Model\EventStream\EventStreamFilter; use Neos\EventStore\Model\EventStream\ExpectedVersion; use Neos\EventStore\Model\EventStream\VirtualStreamName; @@ -37,9 +36,9 @@ class ContentStreamPruner implements ContentRepositoryServiceInterface { public function __construct( - private readonly ContentRepository $contentRepository, private readonly EventStoreInterface $eventStore, - private readonly EventNormalizer $eventNormalizer + private readonly EventNormalizer $eventNormalizer, + private readonly SubscriptionEngine $subscriptionEngine, ) { } @@ -159,10 +158,9 @@ public function removeDanglingContentStreams(\Closure $outputFn, \DateTimeImmuta } if ($danglingContentStreamsPresent) { - try { - $this->contentRepository->catchUpProjections(); - } catch (\Throwable $e) { - $outputFn(sprintf('Could not catchup after removing unused content streams: %s. You might need to use ./flow contentstream:pruneremovedfromeventstream and replay.', $e->getMessage())); + $result = $this->subscriptionEngine->catchUpActive(); + if ($result->hadErrors()) { + $outputFn('Catchup after removing unused content streams led to errors. You might need to use ./flow contentstream:pruneremovedfromeventstream and replay.'); } } else { $outputFn('Okay. No pruneable streams in the event stream'); @@ -198,15 +196,6 @@ public function pruneRemovedFromEventStream(\Closure $outputFn): void } } - public function pruneAllWorkspacesAndContentStreamsFromEventStream(): void - { - foreach ($this->findAllContentStreamStreamNames() as $contentStreamStreamName) { - $this->eventStore->deleteStream($contentStreamStreamName); - } - foreach ($this->findAllWorkspaceStreamNames() as $workspaceStreamName) { - $this->eventStore->deleteStream($workspaceStreamName); - } - } /** * Find all removed content streams that are unused in the event stream @@ -390,48 +379,4 @@ private function findAllContentStreams(): array } return $cs; } - - /** - * @return list - */ - private function findAllContentStreamStreamNames(): array - { - $events = $this->eventStore->load( - VirtualStreamName::forCategory(ContentStreamEventStreamName::EVENT_STREAM_NAME_PREFIX), - EventStreamFilter::create( - EventTypes::create( - // we are only interested in the creation events to limit the amount of events to fetch - EventType::fromString('ContentStreamWasCreated'), - EventType::fromString('ContentStreamWasForked') - ) - ) - ); - $allStreamNames = []; - foreach ($events as $eventEnvelope) { - $allStreamNames[] = $eventEnvelope->streamName; - } - return array_unique($allStreamNames, SORT_REGULAR); - } - - /** - * @return list - */ - private function findAllWorkspaceStreamNames(): array - { - $events = $this->eventStore->load( - VirtualStreamName::forCategory(WorkspaceEventStreamName::EVENT_STREAM_NAME_PREFIX), - EventStreamFilter::create( - EventTypes::create( - // we are only interested in the creation events to limit the amount of events to fetch - EventType::fromString('RootWorkspaceWasCreated'), - EventType::fromString('WorkspaceWasCreated') - ) - ) - ); - $allStreamNames = []; - foreach ($events as $eventEnvelope) { - $allStreamNames[] = $eventEnvelope->streamName; - } - return array_unique($allStreamNames, SORT_REGULAR); - } } diff --git a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php index f9940f8f56a..ecdb4dc2107 100644 --- a/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php +++ b/Neos.ContentRepository.Core/Classes/Service/ContentStreamPrunerFactory.php @@ -16,9 +16,9 @@ class ContentStreamPrunerFactory implements ContentRepositoryServiceFactoryInter public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentStreamPruner { return new ContentStreamPruner( - $serviceFactoryDependencies->contentRepository, $serviceFactoryDependencies->eventStore, - $serviceFactoryDependencies->eventNormalizer + $serviceFactoryDependencies->eventNormalizer, + $serviceFactoryDependencies->subscriptionEngine, ); } } diff --git a/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php b/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php index c7de2e5bb28..db546bf9f9f 100644 --- a/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php +++ b/Neos.ContentRepository.Core/Classes/SharedModel/ContentRepository/ContentRepositoryStatus.php @@ -14,32 +14,42 @@ namespace Neos\ContentRepository\Core\SharedModel\ContentRepository; -use Neos\ContentRepository\Core\Projection\ProjectionStatuses; -use Neos\ContentRepository\Core\Projection\ProjectionStatusType; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatusCollection; +use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventStore\Status as EventStoreStatus; -use Neos\EventStore\Model\EventStore\StatusType as EventStoreStatusType; /** + * The status information of a content repository. Examined via {@see ContentRepositoryMaintainer::status()} + * * @api */ final readonly class ContentRepositoryStatus { - public function __construct( + /** + * @param EventStoreStatus $eventStoreStatus + * @param SequenceNumber|null $eventStorePosition The position of the event store. NULL if an error occurred in which case a setup must likely be done, see $eventStoreStatus + * @param SubscriptionStatusCollection $subscriptionStatus + */ + private function __construct( public EventStoreStatus $eventStoreStatus, - public ProjectionStatuses $projectionStatuses, + public SequenceNumber|null $eventStorePosition, + public SubscriptionStatusCollection $subscriptionStatus, ) { } - public function isOk(): bool - { - if ($this->eventStoreStatus->type !== EventStoreStatusType::OK) { - return false; - } - foreach ($this->projectionStatuses as $projectionStatus) { - if ($projectionStatus->type !== ProjectionStatusType::OK) { - return false; - } - } - return true; + /** + * @internal + */ + public static function create( + EventStoreStatus $eventStoreStatus, + SequenceNumber|null $eventStorePosition, + SubscriptionStatusCollection $subscriptionStatus, + ): self { + return new self( + $eventStoreStatus, + $eventStorePosition, + $subscriptionStatus + ); } } diff --git a/Neos.ContentRepository.Core/Classes/Subscription/DetachedSubscriptionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/DetachedSubscriptionStatus.php new file mode 100644 index 00000000000..7ab2ad36ac0 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/DetachedSubscriptionStatus.php @@ -0,0 +1,34 @@ + + * @internal implementation detail of the catchup + */ +final readonly class Errors implements \IteratorAggregate, \Countable +{ + private const CLAMP_ERRORS = 5; + + /** + * @var non-empty-array + */ + private array $errors; + + private function __construct( + Error ...$errors + ) { + if ($errors === []) { + throw new \InvalidArgumentException('Errors must not be empty.', 1731612542); + } + $this->errors = array_values($errors); + } + + /** + * @param array $errors + */ + public static function fromArray(array $errors): self + { + return new self(...$errors); + } + + public function getIterator(): \Traversable + { + yield from $this->errors; + } + + public function count(): int + { + return count($this->errors); + } + + public function first(): Error + { + foreach ($this->errors as $error) { + return $error; + } + } + + public function getClampedMessage(): string + { + $additionalMessage = ''; + $lines = []; + foreach ($this->errors as $error) { + $lines[] = sprintf('%s"%s": %s', $error->position ? 'Event ' . $error->position->value . ' in ' : '', $error->subscriptionId->value, $error->message); + if (count($lines) >= self::CLAMP_ERRORS) { + $additionalMessage = sprintf('%sAnd %d other exceptions, see log.', ";\n", count($this->errors) - self::CLAMP_ERRORS); + break; + } + } + return join(";\n", $lines) . $additionalMessage; + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php new file mode 100644 index 00000000000..eabe5c5270a --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/ProcessedResult.php @@ -0,0 +1,33 @@ +errors */ + public function hadErrors(): bool + { + return $this->errors !== null; + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php new file mode 100644 index 00000000000..dab71033f97 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/Result.php @@ -0,0 +1,26 @@ +logger?->info('Subscription Engine: Start to setup.'); + + $this->subscriptionStore->setup(); + $this->discoverNewSubscriptions(); + $subscriptions = $this->subscriptionStore->findByCriteriaForUpdate(SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatusFilter::fromArray([ + SubscriptionStatus::NEW, + SubscriptionStatus::BOOTING, + SubscriptionStatus::ACTIVE + ]))); + if ($subscriptions->isEmpty()) { + // should not happen as this means the contentGraph is unavailable, see status information. + $this->logger?->info('Subscription Engine: No subscriptions found.'); + return Result::success(); + } + $errors = []; + foreach ($subscriptions as $subscription) { + $error = $this->setupSubscription($subscription); + if ($error !== null) { + $errors[] = $error; + } + } + return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); + } + + public function boot(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null, int $batchSize = null): ProcessedResult + { + $criteria ??= SubscriptionEngineCriteria::noConstraints(); + return $this->processExclusively( + fn () => $this->catchUpSubscriptions($criteria, SubscriptionStatusFilter::fromArray([SubscriptionStatus::BOOTING]), $progressCallback, $batchSize) + ); + } + + public function catchUpActive(SubscriptionEngineCriteria|null $criteria = null, \Closure $progressCallback = null, int $batchSize = null): ProcessedResult + { + $criteria ??= SubscriptionEngineCriteria::noConstraints(); + return $this->processExclusively( + fn () => $this->catchUpSubscriptions($criteria, SubscriptionStatusFilter::fromArray([SubscriptionStatus::ACTIVE]), $progressCallback, $batchSize) + ); + } + + public function reset(SubscriptionEngineCriteria|null $criteria = null): Result + { + $criteria ??= SubscriptionEngineCriteria::noConstraints(); + + $this->logger?->info('Subscription Engine: Start to reset.'); + $subscriptions = $this->subscriptionStore->findByCriteriaForUpdate(SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, SubscriptionStatusFilter::any())); + if ($subscriptions->isEmpty()) { + $this->logger?->info('Subscription Engine: No subscriptions to reset.'); + return Result::success(); + } + $errors = []; + foreach ($subscriptions as $subscription) { + if ( + $subscription->status === SubscriptionStatus::NEW + || !$this->subscribers->contain($subscription->id) + ) { + // Todo mark projections as detached like setup or catchup? + continue; + } + $error = $this->resetSubscription($subscription); + if ($error !== null) { + $errors[] = $error; + } + } + return $errors === [] ? Result::success() : Result::failed(Errors::fromArray($errors)); + } + + public function subscriptionStatus(SubscriptionEngineCriteria|null $criteria = null): SubscriptionStatusCollection + { + $statuses = []; + try { + $subscriptions = $this->subscriptionStore->findByCriteriaForUpdate(SubscriptionCriteria::create(ids: $criteria?->ids)); + } catch (TableNotFoundException) { + // the schema is not setup - thus there are no subscribers + return SubscriptionStatusCollection::createEmpty(); + } + foreach ($subscriptions as $subscription) { + if (!$this->subscribers->contain($subscription->id)) { + $statuses[] = DetachedSubscriptionStatus::create( + $subscription->id, + $subscription->status, + $subscription->position + ); + continue; + } + $subscriber = $this->subscribers->get($subscription->id); + $statuses[] = ProjectionSubscriptionStatus::create( + subscriptionId: $subscription->id, + subscriptionStatus: $subscription->status, + subscriptionPosition: $subscription->position, + subscriptionError: $subscription->error, + setupStatus: $subscriber->projection->status(), + ); + } + foreach ($this->subscribers as $subscriber) { + if ($subscriptions->contain($subscriber->id)) { + continue; + } + if ($criteria?->ids?->contain($subscriber->id) === false) { + // this might be a NEW subscription but we dont return it as status is filtered. + continue; + } + // this NEW state is not persisted yet + $statuses[] = ProjectionSubscriptionStatus::create( + subscriptionId: $subscriber->id, + subscriptionStatus: SubscriptionStatus::NEW, + subscriptionPosition: SequenceNumber::none(), + subscriptionError: null, + setupStatus: $subscriber->projection->status(), + ); + } + return SubscriptionStatusCollection::fromArray($statuses); + } + + /** + * Find all subscribers that don't have a corresponding subscription. + * For each match a subscription is added + * + * Note: newly discovered subscriptions are not ACTIVE by default, instead they have to be initialized via {@see self::setup()} explicitly + */ + private function discoverNewSubscriptions(): void + { + $subscriptions = $this->subscriptionStore->findByCriteriaForUpdate(SubscriptionCriteria::noConstraints()); + foreach ($this->subscribers as $subscriber) { + if ($subscriptions->contain($subscriber->id)) { + continue; + } + $subscription = new Subscription( + $subscriber->id, + SubscriptionStatus::NEW, + SequenceNumber::fromInteger(0), + null, + null + ); + $this->subscriptionStore->add($subscription); + $this->logger?->info(sprintf('Subscription Engine: New Subscriber "%s" was found and added to the subscription store.', $subscriber->id->value)); + } + } + + /** + * Set up the subscription by retrieving the corresponding subscriber and calling the setUp method on its handler + * If the setup fails, the subscription will be in the {@see SubscriptionStatus::ERROR} state and a corresponding {@see Error} is returned + */ + private function setupSubscription(Subscription $subscription): ?Error + { + if (!$this->subscribers->contain($subscription->id)) { + // mark detached subscriptions as we cannot set up + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::DETACHED, + position: $subscription->position, + subscriptionError: $subscription->error + ); + $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); + return null; + } + + $subscriber = $this->subscribers->get($subscription->id); + try { + $subscriber->projection->setUp(); + } catch (\Throwable $e) { + $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the setup method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); + $this->subscriptionStore->update( + $subscription->id, + SubscriptionStatus::ERROR, + $subscription->position, + SubscriptionError::fromPreviousStatusAndException($subscription->status, $e) + ); + return Error::create($subscription->id, $e->getMessage(), $e, null); + } + + if ($subscription->status === SubscriptionStatus::ACTIVE) { + $this->logger?->debug(sprintf('Subscription Engine: Active subscriber "%s" for "%s" has been re-setup.', $subscriber::class, $subscription->id->value)); + return null; + } else { + $this->subscriptionStore->update( + $subscription->id, + SubscriptionStatus::BOOTING, + $subscription->position, + null + ); + } + $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" has been setup, set to %s from previous %s.', $subscriber::class, $subscription->id->value, SubscriptionStatus::BOOTING->value, $subscription->status->name)); + return null; + } + + private function resetSubscription(Subscription $subscription): ?Error + { + $subscriber = $this->subscribers->get($subscription->id); + try { + $subscriber->projection->resetState(); + } catch (\Throwable $e) { + $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" has an error in the resetState method: %s', $subscriber::class, $subscription->id->value, $e->getMessage())); + return Error::create($subscription->id, $e->getMessage(), $e, null); + } + $this->subscriptionStore->update( + $subscription->id, + SubscriptionStatus::BOOTING, + position: SequenceNumber::none(), + subscriptionError: null + ); + $this->logger?->debug(sprintf('Subscription Engine: For Subscriber "%s" for "%s" the resetState method has been executed.', $subscriber::class, $subscription->id->value)); + return null; + } + + /** + * @param \Closure|null $progressCallback The callback that is invoked for every {@see EventEnvelope} that is processed per subscriber + * @param int|null $batchSize Number of events to process before the transaction is commited and reopened. (defaults to all events). + */ + private function catchUpSubscriptions(SubscriptionEngineCriteria $criteria, SubscriptionStatusFilter $status, \Closure|null $progressCallback, int|null $batchSize): ProcessedResult + { + if ($batchSize !== null && $batchSize <= 0) { + throw new \InvalidArgumentException(sprintf('Invalid batchSize %d specified, must be either NULL or a positive integer.', $batchSize), 1733597950); + } + + $this->logger?->info(sprintf('Subscription Engine: Start catching up subscriptions in states %s.', join(',', $status->toStringArray()))); + + $subscriptionCriteria = SubscriptionCriteria::forEngineCriteriaAndStatus($criteria, $status); + + $numberOfProcessedEvents = 0; + /** @var array $errors */ + $errors = []; + + $this->subscriptionStore->beginTransaction(); + + $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate($subscriptionCriteria); + foreach ($subscriptionsToCatchup as $subscription) { + if (!$this->subscribers->contain($subscription->id)) { + // mark detached subscriptions as we cannot handle them and exclude them from catchup + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::DETACHED, + position: $subscription->position, + subscriptionError: null, + ); + $this->logger?->info(sprintf('Subscription Engine: Subscriber for "%s" not found and has been marked as detached.', $subscription->id->value)); + $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); + } + } + + if ($subscriptionsToCatchup->isEmpty()) { + $this->logger?->info('Subscription Engine: No subscriptions matched criteria. Finishing catch up.'); + $this->subscriptionStore->commit(); + return ProcessedResult::success(0); + } + + $subscriptionIdsToInvokeAroundCatchUpHooks = $subscriptionsToCatchup->getIds(); + foreach ($subscriptionsToCatchup as $subscription) { + $subscriber = $this->subscribers->get($subscription->id); + try { + $subscriber->catchUpHook?->onBeforeCatchUp($subscription->status); + } catch (\Throwable $e) { + $errors[] = $error = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null, position: null); + $this->logCatchupHookError($error); + } + } + + while (true) { + /** + * If batching is enabled, the {@see $continueBatching} flag will indicate that the last run was stopped and continuation is necessary to handle the rest of the events. + * It's possible that batching stops at the last event, in that case the transaction is still reopened to set the active state correctly. + */ + $continueBatching = false; + + $startSequenceNumber = $subscriptionsToCatchup->lowestPosition()?->next() ?? SequenceNumber::none(); + $this->logger?->debug(sprintf('Subscription Engine: Event stream is processed from position %s.', $startSequenceNumber->value)); + + /** @var array $highestSequenceNumberForSubscriber */ + $highestSequenceNumberForSubscriber = []; + + $eventStream = $this->eventStore->load(VirtualStreamName::all())->withMinimumSequenceNumber($startSequenceNumber); + foreach ($eventStream as $eventEnvelope) { + $sequenceNumber = $eventEnvelope->sequenceNumber; + if ($numberOfProcessedEvents > 0) { + $this->logger?->debug(sprintf('Subscription Engine: Current event stream position: %s', $sequenceNumber->value)); + } + if ($progressCallback !== null) { + $progressCallback($eventEnvelope); + } + $domainEvent = $this->eventNormalizer->denormalize($eventEnvelope->event); + foreach ($subscriptionsToCatchup as $subscription) { + if ($subscription->position->value >= $sequenceNumber->value) { + $this->logger?->debug(sprintf('Subscription Engine: Subscription "%s" is farther than the current position (%d >= %d), continue catch up.', $subscription->id->value, $subscription->position->value, $sequenceNumber->value)); + continue; + } + if (!$subscriptionIdsToInvokeAroundCatchUpHooks->contain($subscription->id)) { + $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" with status "%s" was not part of the first batch, continue catch up.', $subscription->id->value, $subscription->status->value)); + continue; + } + $subscriber = $this->subscribers->get($subscription->id); + + try { + $subscriber->catchUpHook?->onBeforeEvent($domainEvent, $eventEnvelope); + } catch (\Throwable $e) { + $errors[] = $error = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null, $eventEnvelope->sequenceNumber); + $this->logCatchupHookError($error); + } + + try { + $subscriber->projection->apply($domainEvent, $eventEnvelope); + } catch (\Throwable $e) { + // ERROR Case: + $errors[] = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null, $eventEnvelope->sequenceNumber); + $this->logger?->error(sprintf('Subscription Engine: Subscriber "%s" for "%s" could not process the event "%s" (sequence number: %d): %s', $subscriber::class, $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value, $e->getMessage())); + + // for the leftover events we are not including this failed subscription for catchup + $subscriptionsToCatchup = $subscriptionsToCatchup->without($subscription->id); + // update the subscription error state on either its unchanged or new position (if some events worked) + // note that the possibly partially applied event will not be rolled back. + $this->subscriptionStore->update( + $subscription->id, + status: SubscriptionStatus::ERROR, + position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, + subscriptionError: SubscriptionError::fromPreviousStatusAndException( + $subscription->status, + $e + ), + ); + continue; + } + // HAPPY Case: + $this->logger?->debug(sprintf('Subscription Engine: Subscriber "%s" for "%s" processed the event "%s" (sequence number: %d).', substr(strrchr($subscriber::class, '\\') ?: '', 1), $subscription->id->value, $eventEnvelope->event->type->value, $eventEnvelope->sequenceNumber->value)); + $highestSequenceNumberForSubscriber[$subscription->id->value] = $eventEnvelope->sequenceNumber; + + try { + $subscriber->catchUpHook?->onAfterEvent($domainEvent, $eventEnvelope); + } catch (\Throwable $e) { + $errors[] = $error = Error::create($subscription->id, $e->getMessage(), $errors === [] ? $e : null, $eventEnvelope->sequenceNumber); + $this->logCatchupHookError($error); + } + } + $numberOfProcessedEvents++; + if ($batchSize !== null && $numberOfProcessedEvents % $batchSize === 0) { + $continueBatching = true; + $this->logger?->info(sprintf('Subscription Engine: Batch completed with %d events', $numberOfProcessedEvents)); + break; + } + } + foreach ($subscriptionsToCatchup as $subscription) { + // after catchup mark all subscriptions as active, so they are triggered automatically now. + // The position will be set to the one the subscriber handled last, or if no events were in the stream, and we booted we keep the persisted position + $this->subscriptionStore->update( + $subscription->id, + status: $continueBatching === false ? SubscriptionStatus::ACTIVE : $subscription->status, + position: $highestSequenceNumberForSubscriber[$subscription->id->value] ?? $subscription->position, + subscriptionError: null, + ); + if ($continueBatching === false && $subscription->status !== SubscriptionStatus::ACTIVE) { + $this->logger?->info(sprintf('Subscription Engine: Subscription "%s" has been set to active after booting', $subscription->id->value)); + } + } + $this->logger?->info(sprintf('Subscription Engine: Finish catch up. %d processed events %d errors.', $numberOfProcessedEvents, count($errors))); + + $this->subscriptionStore->commit(); + + foreach ($subscriptionIdsToInvokeAroundCatchUpHooks as $subscriptionId) { + try { + $this->subscribers->get($subscriptionId)->catchUpHook?->onAfterBatchCompleted(); + } catch (\Throwable $e) { + $errors[] = $error = Error::create($subscriptionId, $e->getMessage(), $errors === [] ? $e : null, position: null); + $this->logCatchupHookError($error); + } + } + + if ($continueBatching === true && $errors === []) { + // start new batch + $this->subscriptionStore->beginTransaction(); + $subscriptionsToCatchup = $this->subscriptionStore->findByCriteriaForUpdate($subscriptionCriteria); + } else { + break; + } + } + + foreach ($subscriptionIdsToInvokeAroundCatchUpHooks as $subscriptionId) { + try { + $this->subscribers->get($subscriptionId)->catchUpHook?->onAfterCatchUp(); + } catch (\Throwable $e) { + $errors[] = $error = Error::create($subscriptionId, $e->getMessage(), $errors === [] ? $e : null, position: null); + $this->logCatchupHookError($error); + } + } + + return $errors === [] ? ProcessedResult::success($numberOfProcessedEvents) : ProcessedResult::failed($numberOfProcessedEvents, Errors::fromArray($errors)); + } + + private function logCatchupHookError(Error $error): void + { + $this->logger?->error( + sprintf('Subscription Engine: Subscription %s has error in catchup hook: %s', $error->subscriptionId->value, $error->message) + ); + } + + /** + * @template T + * @param \Closure(): T $closure + * @return T + */ + private function processExclusively(\Closure $closure): mixed + { + if ($this->processing) { + throw new SubscriptionEngineAlreadyProcessingException('Subscription engine is already processing', 1732714075); + } + $this->processing = true; + try { + return $closure(); + } finally { + $this->processing = false; + } + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php new file mode 100644 index 00000000000..068f5a15a98 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Engine/SubscriptionEngineCriteria.php @@ -0,0 +1,40 @@ +|null $ids + */ + public static function create( + SubscriptionIds|array $ids = null + ): self { + if (is_array($ids)) { + $ids = SubscriptionIds::fromArray($ids); + } + return new self( + $ids + ); + } + + public static function noConstraints(): self + { + return new self( + ids: null + ); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php new file mode 100644 index 00000000000..68fe0eea6bf --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/CatchUpHadErrors.php @@ -0,0 +1,35 @@ +count() > 1 ? 's' : '', $errors->getClampedMessage()), 1732132930, $errors->first()->throwable); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php b/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php new file mode 100644 index 00000000000..4747a062f31 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Exception/SubscriptionEngineAlreadyProcessingException.php @@ -0,0 +1,12 @@ +|null $ids + * @param SubscriptionStatusFilter|null $status + */ + public static function create( + SubscriptionIds|array $ids = null, + SubscriptionStatusFilter $status = null, + ): self { + if (is_array($ids)) { + $ids = SubscriptionIds::fromArray($ids); + } + return new self( + $ids, + $status ?? SubscriptionStatusFilter::any(), + ); + } + + public static function forEngineCriteriaAndStatus( + SubscriptionEngineCriteria $criteria, + SubscriptionStatusFilter|SubscriptionStatus $status, + ): self { + if ($status instanceof SubscriptionStatus) { + $status = SubscriptionStatusFilter::fromArray([$status]); + } + return new self( + $criteria->ids, + $status, + ); + } + + public static function noConstraints(): self + { + return new self( + ids: null, + status: SubscriptionStatusFilter::any(), + ); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php new file mode 100644 index 00000000000..b47c385a7c2 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Store/SubscriptionStoreInterface.php @@ -0,0 +1,35 @@ + $projection + */ + public function __construct( + public SubscriptionId $id, + public ProjectionInterface $projection, + public ?CatchUpHookInterface $catchUpHook + ) { + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php new file mode 100644 index 00000000000..eba25e39a1a --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriber/Subscribers.php @@ -0,0 +1,87 @@ + + * @internal implementation detail of the catchup + */ +final class Subscribers implements \IteratorAggregate, \Countable, \JsonSerializable +{ + /** + * @param array $subscribersById + */ + private function __construct( + private readonly array $subscribersById + ) { + } + + /** + * @param array $subscribers + */ + public static function fromArray(array $subscribers): self + { + $subscribersById = []; + foreach ($subscribers as $subscriber) { + if (!$subscriber instanceof ProjectionSubscriber) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', ProjectionSubscriber::class, get_debug_type($subscriber)), 1721731490); + } + if (array_key_exists($subscriber->id->value, $subscribersById)) { + throw new \InvalidArgumentException(sprintf('Subscriber with id "%s" is already part of this set', $subscriber->id->value), 1721731494); + } + $subscribersById[$subscriber->id->value] = $subscriber; + } + return new self($subscribersById); + } + + public static function none(): self + { + return self::fromArray([]); + } + + public function with(ProjectionSubscriber $subscriber): self + { + return new self([...$this->subscribersById, $subscriber->id->value => $subscriber]); + } + + public function get(SubscriptionId $id): ProjectionSubscriber + { + if (!$this->contain($id)) { + throw new \InvalidArgumentException(sprintf('Subscriber with the subscription id "%s" not found.', $id->value), 1721731490); + } + return $this->subscribersById[$id->value]; + } + + public function contain(SubscriptionId $id): bool + { + return array_key_exists($id->value, $this->subscribersById); + } + + public function getIterator(): \Traversable + { + yield from array_values($this->subscribersById); + } + + public function count(): int + { + return count($this->subscribersById); + } + + /** + * @return iterable + */ + public function jsonSerialize(): iterable + { + return array_values($this->subscribersById); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php new file mode 100644 index 00000000000..ccfb08f07e5 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscription.php @@ -0,0 +1,22 @@ +getMessage(), $previousStatus, $error->getTraceAsString()); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php new file mode 100644 index 00000000000..668c85f0b78 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionId.php @@ -0,0 +1,30 @@ +value === $this->value; + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php new file mode 100644 index 00000000000..9e81ae56ef2 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionIds.php @@ -0,0 +1,77 @@ + + * @internal implementation detail of the catchup + */ +final class SubscriptionIds implements \IteratorAggregate, \Countable, \JsonSerializable +{ + /** + * @param array $subscriptionIdsById + */ + private function __construct( + private readonly array $subscriptionIdsById + ) { + } + + /** + * @param array $ids + */ + public static function fromArray(array $ids): self + { + $subscriptionIdsById = []; + foreach ($ids as $id) { + if (is_string($id)) { + $id = SubscriptionId::fromString($id); + } + if (!$id instanceof SubscriptionId) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', SubscriptionId::class, get_debug_type($id)), 1731580820); + } + if (array_key_exists($id->value, $subscriptionIdsById)) { + throw new \InvalidArgumentException(sprintf('Subscription id "%s" is already part of this set', $id->value), 1731580838); + } + $subscriptionIdsById[$id->value] = $id; + } + return new self($subscriptionIdsById); + } + + public static function none(): self + { + return self::fromArray([]); + } + + public function getIterator(): \Traversable + { + yield from array_values($this->subscriptionIdsById); + } + + public function count(): int + { + return count($this->subscriptionIdsById); + } + + public function contain(SubscriptionId $id): bool + { + return array_key_exists($id->value, $this->subscriptionIdsById); + } + + /** + * @return list + */ + public function toStringArray(): array + { + return array_values(array_map(static fn (SubscriptionId $id) => $id->value, $this->subscriptionIdsById)); + } + + /** + * @return iterable + */ + public function jsonSerialize(): iterable + { + return array_values($this->subscriptionIdsById); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php new file mode 100644 index 00000000000..1af1aa4f1a9 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatus.php @@ -0,0 +1,32 @@ + + */ +final readonly class SubscriptionStatusCollection implements \IteratorAggregate +{ + /** + * @var array $items + */ + private array $items; + + private function __construct( + ProjectionSubscriptionStatus|DetachedSubscriptionStatus ...$items, + ) { + $this->items = $items; + } + + public static function createEmpty(): self + { + return new self(); + } + + /** + * @param array $items + */ + public static function fromArray(array $items): self + { + return new self(...$items); + } + + public function first(): ProjectionSubscriptionStatus|DetachedSubscriptionStatus|null + { + foreach ($this->items as $status) { + return $status; + } + return null; + } + + public function getIterator(): \Traversable + { + yield from $this->items; + } + + public function isEmpty(): bool + { + return $this->items === []; + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php new file mode 100644 index 00000000000..e531c180329 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/SubscriptionStatusFilter.php @@ -0,0 +1,64 @@ + + * @internal implementation detail of the catchup + */ +final class SubscriptionStatusFilter implements \IteratorAggregate +{ + /** + * @param array $statusByValue + */ + private function __construct( + private readonly array $statusByValue, + ) { + } + + /** + * @param array $status + */ + public static function fromArray(array $status): self + { + $statusByValue = []; + foreach ($status as $singleStatus) { + if (is_string($singleStatus)) { + $singleStatus = SubscriptionStatus::from($singleStatus); + } + if (!$singleStatus instanceof SubscriptionStatus) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', SubscriptionStatus::class, get_debug_type($singleStatus)), 1731580994); + } + if (array_key_exists($singleStatus->value, $statusByValue)) { + throw new \InvalidArgumentException(sprintf('Status "%s" is already part of this set', $singleStatus->value), 1731581002); + } + $statusByValue[$singleStatus->value] = $singleStatus; + } + return new self($statusByValue); + } + + public static function any(): self + { + return new self([]); + } + + public function getIterator(): \Traversable + { + yield from array_values($this->statusByValue); + } + + public function isEmpty(): bool + { + return $this->statusByValue === []; + } + + /** + * @return list + */ + public function toStringArray(): array + { + return array_values(array_map(static fn (SubscriptionStatus $id) => $id->value, $this->statusByValue)); + } +} diff --git a/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php new file mode 100644 index 00000000000..f3316a486d8 --- /dev/null +++ b/Neos.ContentRepository.Core/Classes/Subscription/Subscriptions.php @@ -0,0 +1,142 @@ + + * @internal implementation detail of the catchup + */ +final class Subscriptions implements \IteratorAggregate, \Countable, \JsonSerializable +{ + /** + * @param array $subscriptionsById + */ + private function __construct( + private readonly array $subscriptionsById + ) { + } + + /** + * @param array $subscriptions + */ + public static function fromArray(array $subscriptions): self + { + $subscriptionsById = []; + foreach ($subscriptions as $subscription) { + if (!$subscription instanceof Subscription) { + throw new \InvalidArgumentException(sprintf('Expected instance of %s, got: %s', Subscription::class, get_debug_type($subscription)), 1729679774); + } + if (array_key_exists($subscription->id->value, $subscriptionsById)) { + throw new \InvalidArgumentException(sprintf('Subscription with id "%s" is contained multiple times in this set', $subscription->id->value), 1731580354); + } + $subscriptionsById[$subscription->id->value] = $subscription; + } + return new self($subscriptionsById); + } + + public static function none(): self + { + return self::fromArray([]); + } + + public function getIterator(): \Traversable + { + yield from $this->subscriptionsById; + } + + public function isEmpty(): bool + { + return $this->subscriptionsById === []; + } + + public function first(): ?Subscription + { + foreach ($this->subscriptionsById as $subscription) { + return $subscription; + } + return null; + } + + public function count(): int + { + return count($this->subscriptionsById); + } + + public function contain(SubscriptionId $subscriptionId): bool + { + return array_key_exists($subscriptionId->value, $this->subscriptionsById); + } + + public function get(SubscriptionId $subscriptionId): Subscription + { + if (!$this->contain($subscriptionId)) { + throw new \InvalidArgumentException(sprintf('Subscription with id "%s" not part of this set', $subscriptionId->value), 1723567808); + } + return $this->subscriptionsById[$subscriptionId->value]; + } + + public function without(SubscriptionId $subscriptionId): self + { + $subscriptionsById = $this->subscriptionsById; + unset($subscriptionsById[$subscriptionId->value]); + return new self($subscriptionsById); + } + + /** + * @param \Closure(Subscription): bool $callback + */ + public function filter(\Closure $callback): self + { + return self::fromArray(array_filter($this->subscriptionsById, $callback)); + } + + /** + * @template T + * @param \Closure(Subscription): T $callback + * @return array + */ + public function map(\Closure $callback): array + { + return array_map($callback, $this->subscriptionsById); + } + + public function with(Subscription $subscription): self + { + return new self([...$this->subscriptionsById, $subscription->id->value => $subscription]); + } + + public function getIds(): SubscriptionIds + { + return SubscriptionIds::fromArray(array_map( + fn (Subscription $subscription) => $subscription->id, + iterator_to_array($this->subscriptionsById) + )); + } + + /** + * @return iterable + */ + public function jsonSerialize(): iterable + { + return array_values($this->subscriptionsById); + } + + public function lowestPosition(): SequenceNumber|null + { + if ($this->subscriptionsById === []) { + return null; + } + return SequenceNumber::fromInteger( + min( + array_map( + fn (Subscription $subscription) => $subscription->position->value, + $this->subscriptionsById + ) + ) + ); + } +} diff --git a/Neos.ContentRepository.Core/Tests/Unit/Subscription/CatchUpHadErrorsTest.php b/Neos.ContentRepository.Core/Tests/Unit/Subscription/CatchUpHadErrorsTest.php new file mode 100644 index 00000000000..36f2e968e8f --- /dev/null +++ b/Neos.ContentRepository.Core/Tests/Unit/Subscription/CatchUpHadErrorsTest.php @@ -0,0 +1,102 @@ +getMessage(), + $expectedWrappedException, + null + ), + ]); + + // assert shape if the error had been thrown by the cr: + $catchupHadErrors = CatchUpHadErrors::createFromErrors($errors); + self::assertEquals($expectedWrappedException, $catchupHadErrors->getPrevious()); + self::assertEquals('Error while catching up: "Vendor.Package:SecondFakeProjection": Hook "onBeforeCatchup" failed: "SomeHook": This catchup hook is kaputt.', $catchupHadErrors->getMessage()); + } + + public function testWithTwoHookErrors() + { + $exception = new \RuntimeException('This catchup hook is kaputt.'); + + $expectedWrappedException = new CatchUpHookFailed( + 'Hook "onBeforeEvent" failed: "SomeHook": This catchup hook is kaputt.', + 1733243960, + $exception + ); + + $errors = Errors::fromArray([ + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + $expectedWrappedException, + null + ), + Error::create( + SubscriptionId::fromString('Vendor.Package:SecondFakeProjection'), + $expectedWrappedException->getMessage(), + null, + null + ), + ]); + + // assert shape if the error had been thrown by the cr: + $catchupHadErrors = CatchUpHadErrors::createFromErrors($errors); + self::assertEquals($expectedWrappedException, $catchupHadErrors->getPrevious()); + self::assertEquals('Errors while catching up: "Vendor.Package:SecondFakeProjection": Hook "onBeforeEvent" failed: "SomeHook": This catchup hook is kaputt.; +"Vendor.Package:SecondFakeProjection": Hook "onBeforeEvent" failed: "SomeHook": This catchup hook is kaputt.', $catchupHadErrors->getMessage()); + } + + public function testWith10Errors() + { + $exception = new \RuntimeException('Message why A failed'); + $errors = Errors::fromArray([ + Error::create(SubscriptionId::fromString('Vendor.Package:A'), $exception->getMessage(), $exception, null), + Error::create(SubscriptionId::fromString('Vendor.Package:B'), 'Message why B failed', null, SequenceNumber::fromInteger(1)), + Error::create(SubscriptionId::fromString('Vendor.Package:C'), 'Message why C failed', null, SequenceNumber::fromInteger(1)), + Error::create(SubscriptionId::fromString('Vendor.Package:D'), 'Message why D failed', null, SequenceNumber::fromInteger(3)), + Error::create(SubscriptionId::fromString('Vendor.Package:E'), 'Message why E failed', null, null), + Error::create(SubscriptionId::fromString('Vendor.Package:F'), 'Message why F failed', null, null), + Error::create(SubscriptionId::fromString('Vendor.Package:G'), 'Message why G failed', null, null), + Error::create(SubscriptionId::fromString('Vendor.Package:H'), 'Message why H failed', null, null), + Error::create(SubscriptionId::fromString('Vendor.Package:I'), 'Message why I failed', null, null), + Error::create(SubscriptionId::fromString('Vendor.Package:J'), 'Message why J failed', null, null), + ]); + + // assert shape if the error had been thrown by the cr: + $catchupHadErrors = CatchUpHadErrors::createFromErrors($errors); + self::assertEquals($exception, $catchupHadErrors->getPrevious()); + self::assertEquals(<<getMessage()); + } +} diff --git a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php index 7b8ef635cf2..fa8c3b68f4c 100644 --- a/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php +++ b/Neos.ContentRepository.LegacyNodeMigration/Tests/Behavior/Bootstrap/FeatureContext.php @@ -103,24 +103,26 @@ public function iRunTheEventMigration(RootNodeTypeMapping $rootNodeTypeMapping = $propertyMapper = $this->getObject(PropertyMapper::class); // HACK to access the property converter - $propertyConverterAccess = new class implements ContentRepositoryServiceFactoryInterface { + $crInternalsAccess = new class implements ContentRepositoryServiceFactoryInterface { public PropertyConverter|null $propertyConverter; + public EventNormalizer|null $eventNormalizer; public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface { $this->propertyConverter = $serviceFactoryDependencies->propertyConverter; + $this->eventNormalizer = $serviceFactoryDependencies->eventNormalizer; return new class implements ContentRepositoryServiceInterface { }; } }; - $this->getContentRepositoryService($propertyConverterAccess); + $this->getContentRepositoryService($crInternalsAccess); $eventExportProcessor = new EventExportProcessor( $nodeTypeManager, $propertyMapper, - $propertyConverterAccess->propertyConverter, + $crInternalsAccess->propertyConverter, $this->currentContentRepository->getVariationGraph(), - $this->getObject(EventNormalizer::class), + $crInternalsAccess->eventNormalizer, $rootNodeTypeMapping ?? RootNodeTypeMapping::fromArray(['/sites' => NodeTypeNameFactory::NAME_SITES]), $this->nodeDataRows ); diff --git a/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentService.php b/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentService.php index 753525af34f..bf18eaea3b2 100644 --- a/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentService.php +++ b/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentService.php @@ -6,7 +6,7 @@ use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\DimensionSpace\InterDimensionalVariationGraph; -use Neos\ContentRepository\Core\EventStore\EventPersister; +use Neos\ContentRepository\Core\EventStore\EventNormalizer; use Neos\ContentRepository\Core\EventStore\EventsToPublish; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Infrastructure\Property\PropertyConverter; @@ -14,12 +14,15 @@ use Neos\ContentRepository\Core\NodeType\NodeTypeName; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphInterface; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\ContentRepository\StructureAdjustment\Adjustment\DimensionAdjustment; use Neos\ContentRepository\StructureAdjustment\Adjustment\DisallowedChildNodeAdjustment; use Neos\ContentRepository\StructureAdjustment\Adjustment\PropertyAdjustment; use Neos\ContentRepository\StructureAdjustment\Adjustment\StructureAdjustment; use Neos\ContentRepository\StructureAdjustment\Adjustment\TetheredNodeAdjustments; use Neos\ContentRepository\StructureAdjustment\Adjustment\UnknownNodeTypeAdjustment; +use Neos\EventStore\EventStoreInterface; +use Neos\EventStore\Model\Events; class StructureAdjustmentService implements ContentRepositoryServiceInterface { @@ -38,8 +41,10 @@ class StructureAdjustmentService implements ContentRepositoryServiceInterface private readonly ContentGraphInterface $liveContentGraph; public function __construct( - private readonly ContentRepository $contentRepository, - private readonly EventPersister $eventPersister, + ContentRepository $contentRepository, + private readonly EventStoreInterface $eventStore, + private readonly EventNormalizer $eventNormalizer, + private readonly SubscriptionEngine $subscriptionEngine, NodeTypeManager $nodeTypeManager, InterDimensionalVariationGraph $interDimensionalVariationGraph, PropertyConverter $propertyConverter, @@ -98,11 +103,20 @@ public function findAdjustmentsForNodeType(NodeTypeName $nodeTypeName): \Generat public function fixError(StructureAdjustment $adjustment): void { - if ($adjustment->remediation) { - $remediation = $adjustment->remediation; - $eventsToPublish = $remediation(); - assert($eventsToPublish instanceof EventsToPublish); - $this->eventPersister->publishEvents($this->contentRepository, $eventsToPublish); + if (!$adjustment->remediation) { + return; } + $remediation = $adjustment->remediation; + $eventsToPublish = $remediation(); + assert($eventsToPublish instanceof EventsToPublish); + $normalizedEvents = Events::fromArray( + $eventsToPublish->events->map($this->eventNormalizer->normalize(...)) + ); + $this->eventStore->commit( + $eventsToPublish->streamName, + $normalizedEvents, + $eventsToPublish->expectedVersion + ); + $this->subscriptionEngine->catchUpActive(); } } diff --git a/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentServiceFactory.php b/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentServiceFactory.php index b9e75abeaff..534fe279028 100644 --- a/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentServiceFactory.php +++ b/Neos.ContentRepository.StructureAdjustment/src/StructureAdjustmentServiceFactory.php @@ -16,7 +16,9 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor { return new StructureAdjustmentService( $serviceFactoryDependencies->contentRepository, - $serviceFactoryDependencies->eventPersister, + $serviceFactoryDependencies->eventStore, + $serviceFactoryDependencies->eventNormalizer, + $serviceFactoryDependencies->subscriptionEngine, $serviceFactoryDependencies->nodeTypeManager, $serviceFactoryDependencies->interDimensionalVariationGraph, $serviceFactoryDependencies->propertyConverter, diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php index a6cac6dc8b2..c06fa32af4c 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/CRTestSuiteTrait.php @@ -22,18 +22,18 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Feature\NodeModification\Dto\PropertyValuesToWrite; use Neos\ContentRepository\Core\NodeType\NodeTypeName; -use Neos\ContentRepository\Core\Projection\CatchUpOptions; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\Filter\FindSubtreeFilter; use Neos\ContentRepository\Core\Projection\ContentGraph\Filter\NodeType\NodeTypeCriteria; use Neos\ContentRepository\Core\Projection\ContentGraph\Subtree; use Neos\ContentRepository\Core\Projection\ContentGraph\VisibilityConstraints; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\ContentStreamClosing; -use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\NodeCopying; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\NodeCreation; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\NodeModification; use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\Features\NodeMove; @@ -146,7 +146,7 @@ public function iExpectTheGraphProjectionToConsistOfExactlyNodes(int $expectedNu public ContentGraphReadModelInterface|null $instance; public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface { - $this->instance = $serviceFactoryDependencies->projectionsAndCatchUpHooks->contentGraphProjection->getState(); + $this->instance = $serviceFactoryDependencies->contentGraphReadModel; return new class implements ContentRepositoryServiceInterface { }; @@ -256,8 +256,9 @@ abstract protected function getContentRepositoryService( */ public function iReplayTheProjection(string $projectionName): void { - $this->currentContentRepository->resetProjectionState($projectionName); - $this->currentContentRepository->catchUpProjection($projectionName, CatchUpOptions::create()); + $contentRepositoryMaintainer = $this->getContentRepositoryService(new ContentRepositoryMaintainerFactory()); + $result = $contentRepositoryMaintainer->replaySubscription(SubscriptionId::fromString($projectionName)); + Assert::assertNull($result); } protected function deserializeProperties(array $properties): PropertyValuesToWrite diff --git a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/GenericCommandExecutionAndEventPublication.php b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/GenericCommandExecutionAndEventPublication.php index 0f4b3d45fa3..47adad031a5 100644 --- a/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/GenericCommandExecutionAndEventPublication.php +++ b/Neos.ContentRepository.TestSuite/Classes/Behavior/Features/Bootstrap/GenericCommandExecutionAndEventPublication.php @@ -20,9 +20,9 @@ use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePointSet; use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint; use Neos\ContentRepository\Core\EventStore\EventNormalizer; -use Neos\ContentRepository\Core\EventStore\EventPersister; -use Neos\ContentRepository\Core\EventStore\Events; -use Neos\ContentRepository\Core\EventStore\EventsToPublish; +use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; +use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; +use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; use Neos\ContentRepository\Core\Feature\Common\RebasableToOtherWorkspaceInterface; use Neos\ContentRepository\Core\Feature\DimensionSpaceAdjustment\Command\AddDimensionShineThrough; use Neos\ContentRepository\Core\Feature\DimensionSpaceAdjustment\Command\MoveDimensionSpacePoint; @@ -57,10 +57,12 @@ use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Exception\WorkspaceRebaseFailed; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Node\ReferenceName; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; use Neos\EventStore\EventStoreInterface; use Neos\EventStore\Model\Event; use Neos\EventStore\Model\Event\StreamName; use Neos\EventStore\Model\EventEnvelope; +use Neos\EventStore\Model\Events; use Neos\EventStore\Model\EventStream\ExpectedVersion; use Neos\EventStore\Model\EventStream\VirtualStreamName; use Neos\Utility\Arrays; @@ -288,19 +290,23 @@ protected function publishEvent(string $eventType, StreamName $streamName, array Event\EventData::fromString(json_encode($eventPayload)), Event\EventMetadata::fromArray([]) ); - /** @var EventPersister $eventPersister */ - $eventPersister = (new \ReflectionClass($this->currentContentRepository))->getProperty('eventPersister') - ->getValue($this->currentContentRepository); - /** @var EventNormalizer $eventNormalizer */ - $eventNormalizer = (new \ReflectionClass($eventPersister))->getProperty('eventNormalizer') - ->getValue($eventPersister); - $event = $eventNormalizer->denormalize($artificiallyConstructedEvent); - - $eventPersister->publishEvents($this->currentContentRepository, new EventsToPublish( - $streamName, - Events::with($event), - ExpectedVersion::ANY() - )); + + // HACK can be replaced, once https://github.com/neos/neos-development-collection/pull/5341 is merged + $eventStoreAndSubscriptionEngine = new class implements ContentRepositoryServiceFactoryInterface { + public EventStoreInterface|null $eventStore; + public SubscriptionEngine|null $subscriptionEngine; + public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface + { + $this->eventStore = $serviceFactoryDependencies->eventStore; + $this->subscriptionEngine = $serviceFactoryDependencies->subscriptionEngine; + return new class implements ContentRepositoryServiceInterface + { + }; + } + }; + $this->getContentRepositoryService($eventStoreAndSubscriptionEngine); + $eventStoreAndSubscriptionEngine->eventStore->commit($streamName, Events::with($artificiallyConstructedEvent), ExpectedVersion::ANY()); + $eventStoreAndSubscriptionEngine->subscriptionEngine->catchUpActive(); } /** diff --git a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory.php b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory.php new file mode 100644 index 00000000000..a20650ae93e --- /dev/null +++ b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory.php @@ -0,0 +1,32 @@ + + * @internal helper to configure custom catchup hook mocks for testing + */ +final class FakeCatchUpHookFactory implements CatchUpHookFactoryInterface +{ + /** + * @var array + */ + private static array $catchupHooks; + + public function build(CatchUpHookFactoryDependencies $dependencies): CatchUpHookInterface + { + return static::$catchupHooks[spl_object_hash($dependencies->projectionState)] ?? throw new \RuntimeException('No catchup hook defined for Fake.'); + } + + public static function setCatchupHook(ProjectionStateInterface $projectionState, CatchUpHookInterface $catchUpHook): void + { + self::$catchupHooks[spl_object_hash($projectionState)] = $catchUpHook; + } +} diff --git a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory2.php b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory2.php new file mode 100644 index 00000000000..cc5610ed1d9 --- /dev/null +++ b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeCatchUpHookFactory2.php @@ -0,0 +1,32 @@ + + * @internal helper to configure custom catchup hook mocks for testing + */ +final class FakeCatchUpHookFactory2 implements CatchUpHookFactoryInterface +{ + /** + * @var array + */ + private static array $catchupHooks; + + public function build(CatchUpHookFactoryDependencies $dependencies): CatchUpHookInterface + { + return static::$catchupHooks[spl_object_hash($dependencies->projectionState)] ?? throw new \RuntimeException('No catchup hook defined for Fake.'); + } + + public static function setCatchupHook(ProjectionStateInterface $projectionState, CatchUpHookInterface $catchUpHook): void + { + self::$catchupHooks[spl_object_hash($projectionState)] = $catchUpHook; + } +} diff --git a/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php new file mode 100644 index 00000000000..d0e21e2ec5a --- /dev/null +++ b/Neos.ContentRepository.TestSuite/Classes/Fakes/FakeProjectionFactory.php @@ -0,0 +1,37 @@ +>> + * @internal helper to configure custom projection mocks for testing + */ +final class FakeProjectionFactory implements ProjectionFactoryInterface +{ + /** + * @var array> + */ + private static array $projections; + + public function build( + SubscriberFactoryDependencies $projectionFactoryDependencies, + array $options, + ): ProjectionInterface { + return static::$projections[$options['instanceId']] ?? throw new \RuntimeException('No projection defined for Fake.'); + } + + /** + * @param ProjectionInterface $projection + */ + public static function setProjection(string $instanceId, ProjectionInterface $projection): void + { + self::$projections[$instanceId] = $projection; + } +} diff --git a/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php b/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php index c026f755119..e4eeaedd33b 100644 --- a/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php +++ b/Neos.ContentRepositoryRegistry.TestSuite/Classes/Behavior/CRRegistrySubjectProvider.php @@ -13,14 +13,14 @@ * source code. */ -use Doctrine\DBAL\Connection; use Neos\ContentRepository\Core\ContentRepository; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; use Neos\ContentRepositoryRegistry\Exception\ContentRepositoryNotFoundException; -use Neos\EventStore\EventStoreInterface; +use PHPUnit\Framework\Assert; /** * The CR registry subject provider trait for behavioral tests @@ -52,21 +52,18 @@ protected function setUpCRRegistry(): void /** * @Given /^I initialize content repository "([^"]*)"$/ */ - public function iInitializeContentRepository(string $contentRepositoryId): void + public function iInitializeContentRepository(string $rawContentRepositoryId): void { - $contentRepository = $this->getContentRepository(ContentRepositoryId::fromString($contentRepositoryId)); - /** @var EventStoreInterface $eventStore */ - $eventStore = (new \ReflectionClass($contentRepository))->getProperty('eventStore')->getValue($contentRepository); - /** @var Connection $databaseConnection */ - $databaseConnection = (new \ReflectionClass($eventStore))->getProperty('connection')->getValue($eventStore); - $eventTableName = sprintf('cr_%s_events', $contentRepositoryId); - $databaseConnection->executeStatement('TRUNCATE ' . $eventTableName); + $contentRepositoryId = ContentRepositoryId::fromString($rawContentRepositoryId); - if (!in_array($contentRepository->id, self::$alreadySetUpContentRepositories)) { - $contentRepository->setUp(); - self::$alreadySetUpContentRepositories[] = $contentRepository->id; + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + if (!in_array($contentRepositoryId, self::$alreadySetUpContentRepositories)) { + $result = $contentRepositoryMaintainer->setUp(); + Assert::assertNull($result); + self::$alreadySetUpContentRepositories[] = $contentRepositoryId; } - $contentRepository->resetProjectionStates(); + $result = $contentRepositoryMaintainer->prune(); + Assert::assertNull($result); } /** diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php index 38149fcbe74..e312c2827ab 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/CrCommandController.php @@ -3,30 +3,39 @@ namespace Neos\ContentRepositoryRegistry\Command; -use Neos\ContentRepository\Core\Projection\CatchUpOptions; use Neos\ContentRepository\Core\Projection\ProjectionStatusType; -use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; -use Neos\ContentRepository\Core\Service\WorkspaceMaintenanceServiceFactory; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; +use Neos\ContentRepository\Core\Subscription\DetachedSubscriptionStatus; +use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; -use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventStore\StatusType; +use Neos\Flow\Annotations as Flow; use Neos\Flow\Cli\CommandController; -use Neos\Neos\Domain\Service\WorkspaceService; -use Symfony\Component\Console\Helper\ProgressBar; -use Symfony\Component\Console\Output\ConsoleOutput; use Symfony\Component\Console\Output\Output; +/** + * Set up a content repository + * + * *Initialisation* + * + * The command "./flow cr:setup" sets up the content repository like event store and subscription database tables. + * It is non-destructive. + * + * Note that a reset is not implemented here but for the Neos CMS use-case provided via "./flow site:pruneAll" + * + * *Staus information* + * + * The status of the content repository e.g. if a setup is required or if all subscriptions are active and their position + * can be examined with "./flow cr:status" + * + * See also {@see ContentRepositoryMaintainer} for more information. + */ final class CrCommandController extends CommandController { - - public function __construct( - private readonly ContentRepositoryRegistry $contentRepositoryRegistry, - private readonly ProjectionServiceFactory $projectionServiceFactory, - ) { - parent::__construct(); - } + #[Flow\Inject()] + protected ContentRepositoryRegistry $contentRepositoryRegistry; /** * Sets up and checks required dependencies for a Content Repository instance @@ -38,13 +47,22 @@ public function __construct( * To check if the content repository needs to be setup look into cr:status. * That command will also display information what is about to be migrated. * + * @param bool $quiet If set, no output is generated. This is useful if only the exit code (0 = all OK, 1 = errors or warnings) is of interest * @param string $contentRepository Identifier of the Content Repository to set up */ - public function setupCommand(string $contentRepository = 'default'): void + public function setupCommand(string $contentRepository = 'default', bool $quiet = false): void { + if ($quiet) { + $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); + } $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); - $this->contentRepositoryRegistry->get($contentRepositoryId)->setUp(); + $result = $contentRepositoryMaintainer->setUp(); + if ($result !== null) { + $this->outputLine('%s', [$result->getMessage()]); + $this->quit(1); + } $this->outputLine('Content Repository "%s" was set up', [$contentRepositoryId->value]); } @@ -63,35 +81,93 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); } $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $status = $this->contentRepositoryRegistry->get($contentRepositoryId)->status(); - - $this->output('Event Store: '); - $this->outputLine(match ($status->eventStoreStatus->type) { + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + $crStatus = $contentRepositoryMaintainer->status(); + $hasErrors = false; + $replayRequired = false; + $setupRequired = false; + $this->outputLine('Event Store:'); + $this->output(' Setup: '); + $this->outputLine(match ($crStatus->eventStoreStatus->type) { StatusType::OK => 'OK', StatusType::SETUP_REQUIRED => 'Setup required!', StatusType::ERROR => 'ERROR', }); - if ($verbose && $status->eventStoreStatus->details !== '') { - $this->outputFormatted($status->eventStoreStatus->details, [], 2); + if ($crStatus->eventStorePosition) { + $this->outputLine(' Position: %d', [$crStatus->eventStorePosition->value]); + } else { + $this->outputLine(' Position: Loading failed!'); + } + $hasErrors |= $crStatus->eventStoreStatus->type === StatusType::ERROR; + if ($verbose && $crStatus->eventStoreStatus->details !== '') { + $this->outputFormatted($crStatus->eventStoreStatus->details, [], 2); } $this->outputLine(); - foreach ($status->projectionStatuses as $projectionName => $projectionStatus) { - $this->output('Projection "%s": ', [$projectionName]); - $this->outputLine(match ($projectionStatus->type) { - ProjectionStatusType::OK => 'OK', - ProjectionStatusType::SETUP_REQUIRED => 'Setup required!', - ProjectionStatusType::REPLAY_REQUIRED => 'Replay required!', - ProjectionStatusType::ERROR => 'ERROR', - }); - if ($verbose && ($projectionStatus->type !== ProjectionStatusType::OK || $projectionStatus->details)) { - $lines = explode(chr(10), $projectionStatus->details ?: 'No details available.'); - foreach ($lines as $line) { - $this->outputLine(' ' . $line); + $this->outputLine('Subscriptions:'); + if ($crStatus->subscriptionStatus->isEmpty()) { + $this->outputLine('There are no registered subscriptions yet, please run ./flow cr:setup'); + $this->quit(1); + } + foreach ($crStatus->subscriptionStatus as $status) { + if ($status instanceof DetachedSubscriptionStatus) { + $this->outputLine(' %s:', [$status->subscriptionId->value]); + $this->output(' Subscription: '); + $this->output('%s DETACHED', [$status->subscriptionId->value, $status->subscriptionStatus === SubscriptionStatus::DETACHED ? 'is' : 'will be']); + $this->outputLine(' at position %d', [$status->subscriptionPosition->value]); + } + if ($status instanceof ProjectionSubscriptionStatus) { + $this->outputLine(' %s:', [$status->subscriptionId->value]); + $this->output(' Setup: '); + $this->outputLine(match ($status->setupStatus->type) { + ProjectionStatusType::OK => 'OK', + ProjectionStatusType::SETUP_REQUIRED => 'SETUP REQUIRED', + ProjectionStatusType::ERROR => 'ERROR', + }); + $hasErrors |= $status->setupStatus->type === ProjectionStatusType::ERROR; + $setupRequired |= $status->setupStatus->type === ProjectionStatusType::SETUP_REQUIRED; + if ($verbose && ($status->setupStatus->type !== ProjectionStatusType::OK || $status->setupStatus->details)) { + $lines = explode(chr(10), $status->setupStatus->details ?: 'No details available.'); + foreach ($lines as $line) { + $this->outputLine(' ' . $line); + } + $this->outputLine(); + } + $this->output(' Projection: '); + $this->output(match ($status->subscriptionStatus) { + SubscriptionStatus::NEW => 'NEW', + SubscriptionStatus::BOOTING => 'BOOTING', + SubscriptionStatus::ACTIVE => 'ACTIVE', + SubscriptionStatus::DETACHED => 'DETACHED', + SubscriptionStatus::ERROR => 'ERROR', + }); + if ($crStatus->eventStorePosition?->value > $status->subscriptionPosition->value) { + // projection is behind + $this->outputLine(' at position %d', [$status->subscriptionPosition->value]); + } else { + $this->outputLine(' at position %d', [$status->subscriptionPosition->value]); + } + $hasErrors |= $status->subscriptionStatus === SubscriptionStatus::ERROR; + $replayRequired |= $status->subscriptionStatus === SubscriptionStatus::ERROR; + $replayRequired |= $status->subscriptionStatus === SubscriptionStatus::BOOTING; + $replayRequired |= $status->subscriptionStatus === SubscriptionStatus::DETACHED; + if ($verbose && $status->subscriptionError !== null) { + $lines = explode(chr(10), $status->subscriptionError->errorMessage ?: 'No details available.'); + foreach ($lines as $line) { + $this->outputLine(' %s', [$line]); + } } - $this->outputLine(); } } - if (!$status->isOk()) { + if ($verbose) { + $this->outputLine(); + if ($setupRequired) { + $this->outputLine('Setup required, please run ./flow cr:setup'); + } + if ($replayRequired) { + $this->outputLine('Replay needed for BOOTING, ERROR or DETACHED subscriptions, please run ./flow subscription:replay [subscription-id]'); + } + } + if ($hasErrors) { $this->quit(1); } } @@ -99,47 +175,35 @@ public function statusCommand(string $contentRepository = 'default', bool $verbo /** * Replays the specified projection of a Content Repository by resetting its state and performing a full catchup. * - * @param string $projection Full Qualified Class Name or alias of the projection to replay (e.g. "contentStream") + * @param string $projection Identifier of the projection to replay * @param string $contentRepository Identifier of the Content Repository instance to operate on * @param bool $force Replay the projection without confirmation. This may take some time! * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) - * @param int $until Until which sequence number should projections be replayed? useful for debugging + * @internal + * @deprecated with Neos 9 Beta 17, please use ./flow subscription:replay instead */ - public function projectionReplayCommand(string $projection, string $contentRepository = 'default', bool $force = false, bool $quiet = false, int $until = 0): void + public function projectionReplayCommand(string $projection, string $contentRepository = 'default', bool $force = false, bool $quiet = false): void { - if ($quiet) { - $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); - } - $progressBar = new ProgressBar($this->output->getOutput()); - $progressBar->setFormat(' %current%/%max% [%bar%] %percent:3s%% %elapsed:16s%/%estimated:-16s% %memory:6s%'); - if (!$force && $quiet) { - $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay this projection. This may take some time.'); + $subscriptionId = match($projection) { + 'doctrineDbalContentGraph', + 'Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjection' => 'contentGraph', + 'documentUriPathProjection' => 'Neos.Neos:DocumentUriPathProjection', + 'change' => 'Neos.Neos:PendingChangesProjection', + default => null + }; + if ($subscriptionId === null) { + $this->outputLine('Invalid --projection specified. Please use ./flow subscription:replay [contentGraph|Neos.Neos:DocumentUriPathProjection|...] directly.'); $this->quit(1); } - - if (!$force && !$this->output->askConfirmation(sprintf('> This will replay the projection "%s" in "%s", which may take some time. Are you sure to proceed? (y/n) ', $projection, $contentRepository), false)) { - $this->outputLine('Abort.'); - return; - } - - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $projectionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionServiceFactory); - - $options = CatchUpOptions::create(); - if (!$quiet) { - $this->outputLine('Replaying events for projection "%s" of Content Repository "%s" ...', [$projection, $contentRepositoryId->value]); - $progressBar->start(max($until > 0 ? $until : $projectionService->highestSequenceNumber()->value, 1)); - $options = $options->with(progressCallback: fn () => $progressBar->advance()); - } - if ($until > 0) { - $options = $options->with(maximumSequenceNumber: SequenceNumber::fromInteger($until)); - } - $projectionService->replayProjection($projection, $options); - if (!$quiet) { - $progressBar->finish(); - $this->outputLine(); - $this->outputLine('Done.'); - } + $this->outputLine('Please use ./flow subscription:replay %s instead!', [$subscriptionId]); + $this->forward( + 'replay', + SubscriptionCommandController::class, + array_merge( + ['subscription' => $subscriptionId], + compact('contentRepository', 'force', 'quiet') + ) + ); } /** @@ -148,61 +212,16 @@ public function projectionReplayCommand(string $projection, string $contentRepos * @param string $contentRepository Identifier of the Content Repository instance to operate on * @param bool $force Replay the projection without confirmation. This may take some time! * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) - * @param int $until Until which sequence number should projections be replayed? useful for debugging + * @internal + * @deprecated with Neos 9 Beta 17, please use ./flow subscription:replayall instead */ - public function projectionReplayAllCommand(string $contentRepository = 'default', bool $force = false, bool $quiet = false, int $until = 0): void + public function projectionReplayAllCommand(string $contentRepository = 'default', bool $force = false, bool $quiet = false): void { - if ($quiet) { - $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); - } - $mainSection = ($this->output->getOutput() instanceof ConsoleOutput) ? $this->output->getOutput()->section() : $this->output->getOutput(); - $mainProgressBar = new ProgressBar($mainSection); - $mainProgressBar->setBarCharacter('â–ˆ'); - $mainProgressBar->setEmptyBarCharacter('â–‘'); - $mainProgressBar->setProgressCharacter('â–ˆ'); - $mainProgressBar->setFormat('debug'); - - $subSection = ($this->output->getOutput() instanceof ConsoleOutput) ? $this->output->getOutput()->section() : $this->output->getOutput(); - $progressBar = new ProgressBar($subSection); - $progressBar->setFormat(' %message% - %current%/%max% [%bar%] %percent:3s%% %elapsed:16s%/%estimated:-16s% %memory:6s%'); - if (!$force && $quiet) { - $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay this projection. This may take some time.'); - $this->quit(1); - } - - if (!$force && !$this->output->askConfirmation(sprintf('> This will replay all projections in "%s", which may take some time. Are you sure to proceed? (y/n) ', $contentRepository), false)) { - $this->outputLine('Abort.'); - return; - } - - $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); - $projectionService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->projectionServiceFactory); - if (!$quiet) { - $this->outputLine('Replaying events for all projections of Content Repository "%s" ...', [$contentRepositoryId->value]); - } - $options = CatchUpOptions::create(); - if (!$quiet) { - $options = $options->with(progressCallback: fn () => $progressBar->advance()); - } - if ($until > 0) { - $options = $options->with(maximumSequenceNumber: SequenceNumber::fromInteger($until)); - } - $highestSequenceNumber = max($until > 0 ? $until : $projectionService->highestSequenceNumber()->value, 1); - $mainProgressBar->start($projectionService->numberOfProjections()); - $mainProgressCallback = null; - if (!$quiet) { - $mainProgressCallback = static function (string $projectionAlias) use ($mainProgressBar, $progressBar, $highestSequenceNumber) { - $mainProgressBar->advance(); - $progressBar->setMessage($projectionAlias); - $progressBar->start($highestSequenceNumber); - $progressBar->setProgress(0); - }; - } - $projectionService->replayAllProjections($options, $mainProgressCallback); - if (!$quiet) { - $mainProgressBar->finish(); - $progressBar->finish(); - $this->outputLine('Done.'); - } + $this->outputLine('Please use ./flow subscription:replayall instead!'); + $this->forward( + 'replayall', + SubscriptionCommandController::class, + compact('contentRepository', 'force', 'quiet') + ); } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/MigrateEventsCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/MigrateEventsCommandController.php index 01cbc8ccf87..821da7d8dc3 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Command/MigrateEventsCommandController.php +++ b/Neos.ContentRepositoryRegistry/Classes/Command/MigrateEventsCommandController.php @@ -168,4 +168,20 @@ public function copyNodesStatusCommand(string $contentRepository = 'default'): v $eventMigrationService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->eventMigrationServiceFactory); $eventMigrationService->copyNodesStatus($this->outputLine(...)); } + + /** + * Migrates the checkpoint tables to ACTIVE subscriptions + * + * Needed for #5321: https://github.com/neos/neos-development-collection/pull/5321 + * + * Included in November 2024 - before final Neos 9.0 release + * + * @param string $contentRepository Identifier of the Content Repository to migrate + */ + public function migrateCheckpointsToSubscriptionsCommand(string $contentRepository = 'default'): void + { + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $eventMigrationService = $this->contentRepositoryRegistry->buildService($contentRepositoryId, $this->eventMigrationServiceFactory); + $eventMigrationService->migrateCheckpointsToSubscriptions($this->outputLine(...)); + } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Command/SubscriptionCommandController.php b/Neos.ContentRepositoryRegistry/Classes/Command/SubscriptionCommandController.php new file mode 100644 index 00000000000..d8ceb350f81 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Command/SubscriptionCommandController.php @@ -0,0 +1,141 @@ +output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); + } + if (!$force && $quiet) { + $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay this subscription. This may take some time.'); + $this->quit(1); + } + + if (!$force && !$this->output->askConfirmation(sprintf('> This will replay the subscription "%s" in "%s", which may take some time. Are you sure to proceed? (y/n) ', $subscription, $contentRepository), false)) { + $this->outputLine('Abort.'); + return; + } + + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + + $progressCallback = null; + if (!$quiet) { + $this->outputLine('Replaying events for subscription "%s" of Content Repository "%s" ...', [$subscription, $contentRepositoryId->value]); + // render memory consumption and time remaining + $this->output->getProgressBar()->setFormat('debug'); + $this->output->progressStart(); + $progressCallback = fn () => $this->output->progressAdvance(); + } + + $result = $contentRepositoryMaintainer->replaySubscription(SubscriptionId::fromString($subscription), progressCallback: $progressCallback); + + if (!$quiet) { + $this->output->progressFinish(); + $this->outputLine(); + } + + if ($result !== null) { + $this->outputLine('%s', [$result->getMessage()]); + $this->quit(1); + } elseif (!$quiet) { + $this->outputLine('Done.'); + } + } + + /** + * Replays all projections of the specified Content Repository by resetting their states and performing a full catchup + * + * @param string $contentRepository Identifier of the Content Repository instance to operate on + * @param bool $force Replay all subscriptions without confirmation. This may take some time! + * @param bool $quiet If set only fatal errors are rendered to the output (must be used with --force flag to avoid user input) + */ + public function replayAllCommand(string $contentRepository = 'default', bool $force = false, bool $quiet = false): void + { + if ($quiet) { + $this->output->getOutput()->setVerbosity(Output::VERBOSITY_QUIET); + } + + if (!$force && $quiet) { + $this->outputLine('Cannot run in quiet mode without --force. Please acknowledge that this command will reset and replay all subscriptions. This may take some time.'); + $this->quit(1); + } + + if (!$force && !$this->output->askConfirmation(sprintf('> This will replay all projections in "%s", which may take some time. Are you sure to proceed? (y/n) ', $contentRepository), false)) { + $this->outputLine('Abort.'); + return; + } + + $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + + $progressCallback = null; + if (!$quiet) { + $this->outputLine('Replaying events for all projections of Content Repository "%s" ...', [$contentRepositoryId->value]); + // render memory consumption and time remaining + // todo maybe reintroduce pretty output: https://github.com/neos/neos-development-collection/pull/5010 but without using highestSequenceNumber + $this->output->getProgressBar()->setFormat('debug'); + $this->output->progressStart(); + $progressCallback = fn () => $this->output->progressAdvance(); + } + + $result = $contentRepositoryMaintainer->replayAllSubscriptions(progressCallback: $progressCallback); + + if (!$quiet) { + $this->output->progressFinish(); + $this->outputLine(); + } + + if ($result !== null) { + $this->outputLine('%s', [$result->getMessage()]); + $this->quit(1); + } elseif (!$quiet) { + $this->outputLine('Done.'); + } + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php index 86804d3cd93..4fdec55cd73 100644 --- a/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php +++ b/Neos.ContentRepositoryRegistry/Classes/ContentRepositoryRegistry.php @@ -11,17 +11,21 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryFactory; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; -use Neos\ContentRepository\Core\Factory\ProjectionsAndCatchUpHooksFactory; +use Neos\ContentRepository\Core\Factory\ContentRepositorySubscriberFactories; +use Neos\ContentRepository\Core\Factory\ProjectionSubscriberFactory; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactories; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionFactoryInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentSubgraphInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\Node; use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; -use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStateInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryIds; +use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; use Neos\ContentRepositoryRegistry\Exception\ContentRepositoryNotFoundException; use Neos\ContentRepositoryRegistry\Exception\InvalidConfigurationException; use Neos\ContentRepositoryRegistry\Factory\AuthProvider\AuthProviderFactoryInterface; @@ -29,6 +33,7 @@ use Neos\ContentRepositoryRegistry\Factory\ContentDimensionSource\ContentDimensionSourceFactoryInterface; use Neos\ContentRepositoryRegistry\Factory\EventStore\EventStoreFactoryInterface; use Neos\ContentRepositoryRegistry\Factory\NodeTypeManager\NodeTypeManagerFactoryInterface; +use Neos\ContentRepositoryRegistry\Factory\SubscriptionStore\SubscriptionStoreFactoryInterface; use Neos\ContentRepositoryRegistry\SubgraphCachingInMemory\ContentSubgraphWithRuntimeCaches; use Neos\ContentRepositoryRegistry\SubgraphCachingInMemory\SubgraphCachePool; use Neos\EventStore\EventStoreInterface; @@ -37,6 +42,7 @@ use Neos\Utility\Arrays; use Neos\Utility\PositionalArraySorter; use Psr\Clock\ClockInterface; +use Psr\Log\LoggerInterface; use Symfony\Component\Serializer\Normalizer\DenormalizerInterface; use Symfony\Component\Serializer\Normalizer\NormalizerInterface; use Symfony\Component\Serializer\Serializer; @@ -44,7 +50,7 @@ /** * @api */ -#[Flow\Scope("singleton")] +#[Flow\Scope('singleton')] final class ContentRepositoryRegistry { /** @@ -53,13 +59,26 @@ final class ContentRepositoryRegistry private array $factoryInstances = []; /** + * @var array + */ + private array $settings; + + #[Flow\Inject(name: 'Neos.ContentRepositoryRegistry:Logger', lazy: false)] + protected LoggerInterface $logger; + + #[Flow\Inject()] + protected ObjectManagerInterface $objectManager; + + #[Flow\Inject()] + protected SubgraphCachePool $subgraphCachePool; + + /** + * @internal for flow wiring and test cases only * @param array $settings */ - public function __construct( - private readonly array $settings, - private readonly ObjectManagerInterface $objectManager, - private readonly SubgraphCachePool $subgraphCachePool, - ) { + public function injectSettings(array $settings): void + { + $this->settings = $settings; } /** @@ -94,16 +113,6 @@ public function getContentRepositoryIds(): ContentRepositoryIds return ContentRepositoryIds::fromArray($contentRepositoryIds); } - /** - * @internal for test cases only - */ - public function resetFactoryInstance(ContentRepositoryId $contentRepositoryId): void - { - if (array_key_exists($contentRepositoryId->value, $this->factoryInstances)) { - unset($this->factoryInstances[$contentRepositoryId->value]); - } - } - public function subgraphForNode(Node $node): ContentSubgraphInterface { $contentRepository = $this->get($node->contentRepositoryId); @@ -132,6 +141,16 @@ public function buildService(ContentRepositoryId $contentRepositoryId, ContentRe return $this->getFactory($contentRepositoryId)->buildService($contentRepositoryServiceFactory); } + /** + * @internal for test cases only + */ + public function resetFactoryInstance(ContentRepositoryId $contentRepositoryId): void + { + if (array_key_exists($contentRepositoryId->value, $this->factoryInstances)) { + unset($this->factoryInstances[$contentRepositoryId->value]); + } + } + /** * @throws ContentRepositoryNotFoundException | InvalidConfigurationException */ @@ -168,6 +187,8 @@ private function buildFactory(ContentRepositoryId $contentRepositoryId): Content unset($contentRepositorySettings['preset']); } try { + /** @var CatchUpHookFactoryInterface|null $contentGraphCatchUpHookFactory */ + $contentGraphCatchUpHookFactory = $this->buildCatchUpHookFactory($contentRepositoryId, 'contentGraph', $contentRepositorySettings['contentGraphProjection']); $clock = $this->buildClock($contentRepositoryId, $contentRepositorySettings); return new ContentRepositoryFactory( $contentRepositoryId, @@ -175,10 +196,14 @@ private function buildFactory(ContentRepositoryId $contentRepositoryId): Content $this->buildNodeTypeManager($contentRepositoryId, $contentRepositorySettings), $this->buildContentDimensionSource($contentRepositoryId, $contentRepositorySettings), $this->buildPropertySerializer($contentRepositoryId, $contentRepositorySettings), - $this->buildProjectionsFactory($contentRepositoryId, $contentRepositorySettings), $this->buildAuthProviderFactory($contentRepositoryId, $contentRepositorySettings), $clock, + $this->buildSubscriptionStore($contentRepositoryId, $clock, $contentRepositorySettings), + $this->buildContentGraphProjectionFactory($contentRepositoryId, $contentRepositorySettings), + $contentGraphCatchUpHookFactory, $this->buildCommandHooksFactory($contentRepositoryId, $contentRepositorySettings), + $this->buildAdditionalSubscribersFactories($contentRepositoryId, $contentRepositorySettings), + $this->logger, ); } catch (\Exception $exception) { throw InvalidConfigurationException::fromException($contentRepositoryId, $exception); @@ -242,39 +267,44 @@ private function buildPropertySerializer(ContentRepositoryId $contentRepositoryI } /** @param array $contentRepositorySettings */ - private function buildProjectionsFactory(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): ProjectionsAndCatchUpHooksFactory + private function buildContentGraphProjectionFactory(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): ContentGraphProjectionFactoryInterface { - $projectionsAndCatchUpHooksFactory = new ProjectionsAndCatchUpHooksFactory(); - - // content graph projection: if (!isset($contentRepositorySettings['contentGraphProjection']['factoryObjectName'])) { throw InvalidConfigurationException::fromMessage('Content repository "%s" does not have the contentGraphProjection.factoryObjectName configured.', $contentRepositoryId->value); } - $projectionFactory = $this->objectManager->get($contentRepositorySettings['contentGraphProjection']['factoryObjectName']); - if (!$projectionFactory instanceof ContentGraphProjectionFactoryInterface) { - throw InvalidConfigurationException::fromMessage('Projection factory object name of contentGraphProjection (content repository "%s") is not an instance of %s but %s.', $contentRepositoryId->value, ContentGraphProjectionFactoryInterface::class, get_debug_type($projectionFactory)); + $contentGraphProjectionFactory = $this->objectManager->get($contentRepositorySettings['contentGraphProjection']['factoryObjectName']); + if (!$contentGraphProjectionFactory instanceof ContentGraphProjectionFactoryInterface) { + throw InvalidConfigurationException::fromMessage('Projection factory object name of contentGraphProjection (content repository "%s") is not an instance of %s but %s.', $contentRepositoryId->value, ContentGraphProjectionFactoryInterface::class, get_debug_type($contentGraphProjectionFactory)); } - $projectionsAndCatchUpHooksFactory->registerFactory($projectionFactory, $contentRepositorySettings['contentGraphProjection']['options'] ?? []); - - $this->registerCatchupHookForProjection($contentRepositorySettings['contentGraphProjection'], $projectionsAndCatchUpHooksFactory, $projectionFactory, 'contentGraphProjection', $contentRepositoryId); + return $contentGraphProjectionFactory; + } - // additional projections: - (is_array($contentRepositorySettings['projections'] ?? [])) || throw InvalidConfigurationException::fromMessage('Content repository "%s" expects projections configured as array.', $contentRepositoryId->value); - foreach ($contentRepositorySettings['projections'] ?? [] as $projectionName => $projectionOptions) { - if ($projectionOptions === null) { + /** + * @param array $projectionOptions + * @return CatchUpHookFactoryInterface|null + */ + private function buildCatchUpHookFactory(ContentRepositoryId $contentRepositoryId, string $projectionName, array $projectionOptions): ?CatchUpHookFactoryInterface + { + if (!isset($projectionOptions['catchUpHooks'])) { + return null; + } + $catchUpHookFactories = CatchUpHookFactories::create(); + foreach ($projectionOptions['catchUpHooks'] as $catchUpHookName => $catchUpHookOptions) { + if ($catchUpHookOptions === null) { + // Allow catch up hooks to be disabled by setting their configuration to `null` continue; } - (is_array($projectionOptions)) || throw InvalidConfigurationException::fromMessage('Projection "%s" (content repository "%s") must be configured as array got %s', $projectionName, $contentRepositoryId->value, get_debug_type($projectionOptions)); - $projectionFactory = isset($projectionOptions['factoryObjectName']) ? $this->objectManager->get($projectionOptions['factoryObjectName']) : null; - if (!$projectionFactory instanceof ProjectionFactoryInterface) { - throw InvalidConfigurationException::fromMessage('Projection factory object name for projection "%s" (content repository "%s") is not an instance of %s but %s.', $projectionName, $contentRepositoryId->value, ProjectionFactoryInterface::class, get_debug_type($projectionFactory)); + $catchUpHookFactory = $this->objectManager->get($catchUpHookOptions['factoryObjectName']); + if (!$catchUpHookFactory instanceof CatchUpHookFactoryInterface) { + throw InvalidConfigurationException::fromMessage('CatchUpHook factory object name for hook "%s" in projection "%s" (content repository "%s") is not an instance of %s but %s', $catchUpHookName, $projectionName, $contentRepositoryId->value, CatchUpHookFactoryInterface::class, get_debug_type($catchUpHookFactory)); } - $projectionsAndCatchUpHooksFactory->registerFactory($projectionFactory, $projectionOptions['options'] ?? []); - - $this->registerCatchupHookForProjection($projectionOptions, $projectionsAndCatchUpHooksFactory, $projectionFactory, $projectionName, $contentRepositoryId); + $catchUpHookFactories = $catchUpHookFactories->with($catchUpHookFactory); + } + if ($catchUpHookFactories->isEmpty()) { + return null; } - return $projectionsAndCatchUpHooksFactory; + return $catchUpHookFactories; } /** @param array $contentRepositorySettings */ @@ -299,21 +329,34 @@ private function buildCommandHooksFactory(ContentRepositoryId $contentRepository return new CommandHooksFactory(...$commandHookFactories); } - /** - * @param ProjectionFactoryInterface> $projectionFactory - */ - private function registerCatchupHookForProjection(mixed $projectionOptions, ProjectionsAndCatchUpHooksFactory $projectionsAndCatchUpHooksFactory, ProjectionFactoryInterface $projectionFactory, string $projectionName, ContentRepositoryId $contentRepositoryId): void + /** @param array $contentRepositorySettings */ + private function buildAdditionalSubscribersFactories(ContentRepositoryId $contentRepositoryId, array $contentRepositorySettings): ContentRepositorySubscriberFactories { - foreach (($projectionOptions['catchUpHooks'] ?? []) as $catchUpHookOptions) { - if ($catchUpHookOptions === null) { + if (!is_array($contentRepositorySettings['projections'] ?? [])) { + throw InvalidConfigurationException::fromMessage('Content repository "%s" expects projections configured as array.', $contentRepositoryId->value); + } + /** @var array $projectionSubscriberFactories */ + $projectionSubscriberFactories = []; + foreach (($contentRepositorySettings['projections'] ?? []) as $projectionName => $projectionOptions) { + // Allow projections to be disabled by setting their configuration to `null` + if ($projectionOptions === null) { continue; } - $catchUpHookFactory = $this->objectManager->get($catchUpHookOptions['factoryObjectName']); - if (!$catchUpHookFactory instanceof CatchUpHookFactoryInterface) { - throw InvalidConfigurationException::fromMessage('CatchUpHook factory object name for projection "%s" (content repository "%s") is not an instance of %s but %s', $projectionName, $contentRepositoryId->value, CatchUpHookFactoryInterface::class, get_debug_type($catchUpHookFactory)); + if (!is_array($projectionOptions)) { + throw InvalidConfigurationException::fromMessage('Projection "%s" (content repository "%s") must be configured as array got %s', $projectionName, $contentRepositoryId->value, get_debug_type($projectionOptions)); + } + $projectionFactory = isset($projectionOptions['factoryObjectName']) ? $this->objectManager->get($projectionOptions['factoryObjectName']) : null; + if (!$projectionFactory instanceof ProjectionFactoryInterface) { + throw InvalidConfigurationException::fromMessage('Projection factory object name for projection "%s" (content repository "%s") is not an instance of %s but %s.', $projectionName, $contentRepositoryId->value, ProjectionFactoryInterface::class, get_debug_type($projectionFactory)); } - $projectionsAndCatchUpHooksFactory->registerCatchUpHookFactory($projectionFactory, $catchUpHookFactory); + $projectionSubscriberFactories[$projectionName] = new ProjectionSubscriberFactory( + SubscriptionId::fromString($projectionName), + $projectionFactory, + $this->buildCatchUpHookFactory($contentRepositoryId, $projectionName, $projectionOptions), + $projectionOptions['options'] ?? [], + ); } + return ContentRepositorySubscriberFactories::fromArray($projectionSubscriberFactories); } /** @param array $contentRepositorySettings */ @@ -337,4 +380,16 @@ private function buildClock(ContentRepositoryId $contentRepositoryIdentifier, ar } return $clockFactory->build($contentRepositoryIdentifier, $contentRepositorySettings['clock']['options'] ?? []); } + + /** @param array $contentRepositorySettings */ + private function buildSubscriptionStore(ContentRepositoryId $contentRepositoryId, ClockInterface $clock, array $contentRepositorySettings): SubscriptionStoreInterface + { + isset($contentRepositorySettings['subscriptionStore']['factoryObjectName']) || throw InvalidConfigurationException::fromMessage('Content repository "%s" does not have subscriptionStore.factoryObjectName configured.', $contentRepositoryId->value); + $subscriptionStoreFactory = $this->objectManager->get($contentRepositorySettings['subscriptionStore']['factoryObjectName']); + if (!$subscriptionStoreFactory instanceof SubscriptionStoreFactoryInterface) { + throw InvalidConfigurationException::fromMessage('subscriptionStore.factoryObjectName for content repository "%s" is not an instance of %s but %s.', $contentRepositoryId->value, SubscriptionStoreFactoryInterface::class, get_debug_type($subscriptionStoreFactory)); + } + return $subscriptionStoreFactory->build($contentRepositoryId, $clock, $contentRepositorySettings['subscriptionStore']['options'] ?? []); + } } + diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php new file mode 100644 index 00000000000..525f9bd2639 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php @@ -0,0 +1,182 @@ +dbal->createSchemaManager()->createSchemaConfig(); + $schemaConfig->setDefaultTableOptions([ + 'charset' => 'utf8mb4' + ]); + $tableSchema = new Table($this->tableName, [ + (new Column('id', Type::getType(Types::STRING)))->setNotnull(true)->setLength(SubscriptionId::MAX_LENGTH), + (new Column('position', Type::getType(Types::INTEGER)))->setNotnull(true), + (new Column('status', Type::getType(Types::STRING)))->setNotnull(true)->setLength(32), + (new Column('error_message', Type::getType(Types::TEXT)))->setNotnull(false), + (new Column('error_previous_status', Type::getType(Types::STRING)))->setNotnull(false)->setLength(32), + (new Column('error_trace', Type::getType(Types::TEXT)))->setNotnull(false), + (new Column('last_saved_at', Type::getType(Types::DATETIME_IMMUTABLE)))->setNotnull(true), + ]); + $tableSchema->setPrimaryKey(['id']); + $tableSchema->addIndex(['status']); + $schema = new Schema( + [$tableSchema], + [], + $schemaConfig, + ); + foreach (DbalSchemaDiff::determineRequiredSqlStatements($this->dbal, $schema) as $statement) { + $this->dbal->executeStatement($statement); + } + } + + public function findByCriteriaForUpdate(SubscriptionCriteria $criteria): Subscriptions + { + $queryBuilder = $this->dbal->createQueryBuilder() + ->select('*') + ->from($this->tableName) + ->orderBy('id'); + if (!$this->dbal->getDatabasePlatform() instanceof SqlitePlatform) { + $queryBuilder->forUpdate(); + } + if ($criteria->ids !== null) { + $queryBuilder->andWhere('id IN (:ids)') + ->setParameter( + 'ids', + $criteria->ids->toStringArray(), + ArrayParameterType::STRING, + ); + } + if (!$criteria->status->isEmpty()) { + $queryBuilder->andWhere('status IN (:status)') + ->setParameter( + 'status', + $criteria->status->toStringArray(), + ArrayParameterType::STRING, + ); + } + $result = $queryBuilder->executeQuery(); + assert($result instanceof Result); + $rows = $result->fetchAllAssociative(); + if ($rows === []) { + return Subscriptions::none(); + } + return Subscriptions::fromArray(array_map(self::fromDatabase(...), $rows)); + } + + public function add(Subscription $subscription): void + { + $row = self::toDatabase($subscription); + $row['id'] = $subscription->id->value; + $row['last_saved_at'] = $this->clock->now()->format('Y-m-d H:i:s'); + $this->dbal->insert( + $this->tableName, + $row, + ); + } + + public function update( + SubscriptionId $subscriptionId, + SubscriptionStatus $status, + SequenceNumber $position, + SubscriptionError|null $subscriptionError, + ): void { + $row = []; + $row['last_saved_at'] = $this->clock->now()->format('Y-m-d H:i:s'); + $row['status'] = $status->value; + $row['position'] = $position->value; + $row['error_message'] = $subscriptionError?->errorMessage; + $row['error_previous_status'] = $subscriptionError?->previousStatus?->value; + $row['error_trace'] = $subscriptionError?->errorTrace; + $this->dbal->update( + $this->tableName, + $row, + [ + 'id' => $subscriptionId->value, + ] + ); + } + + /** + * @return array + */ + private static function toDatabase(Subscription $subscription): array + { + return [ + 'status' => $subscription->status->value, + 'position' => $subscription->position->value, + 'error_message' => $subscription->error?->errorMessage, + 'error_previous_status' => $subscription->error?->previousStatus?->value, + 'error_trace' => $subscription->error?->errorTrace, + ]; + } + + /** + * @param array $row + */ + private static function fromDatabase(array $row): Subscription + { + if (isset($row['error_message'])) { + $subscriptionError = new SubscriptionError($row['error_message'], SubscriptionStatus::from($row['error_previous_status']), $row['error_trace']); + } else { + $subscriptionError = null; + } + $lastSavedAt = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', $row['last_saved_at']); + if ($lastSavedAt === false) { + throw new \RuntimeException(sprintf('last_saved_at %s is not a valid date', $row['last_saved_at']), 1733602968); + } + + return new Subscription( + SubscriptionId::fromString($row['id']), + SubscriptionStatus::from($row['status']), + SequenceNumber::fromInteger($row['position']), + $subscriptionError, + $lastSavedAt, + ); + } + + public function beginTransaction(): void + { + $this->dbal->beginTransaction(); + } + + public function commit(): void + { + $this->dbal->commit(); + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactory.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactory.php new file mode 100644 index 00000000000..ef1405837e3 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactory.php @@ -0,0 +1,27 @@ + $options */ + public function build(ContentRepositoryId $contentRepositoryId, ClockInterface $clock, array $options): SubscriptionStoreInterface + { + return new DoctrineSubscriptionStore(sprintf('cr_%s_subscriptions', $contentRepositoryId->value), $this->connection, $clock); + } +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactoryInterface.php b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactoryInterface.php new file mode 100644 index 00000000000..79e3516887c --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/SubscriptionStoreFactoryInterface.php @@ -0,0 +1,18 @@ + $options */ + public function build(ContentRepositoryId $contentRepositoryId, ClockInterface $clock, array $options): SubscriptionStoreInterface; +} diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php deleted file mode 100644 index 69587bb4806..00000000000 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionCatchupProcessor.php +++ /dev/null @@ -1,25 +0,0 @@ -projectionservice->catchupAllProjections(CatchUpOptions::create()); - } -} diff --git a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php b/Neos.ContentRepositoryRegistry/Classes/Processors/SubscriptionReplayProcessor.php similarity index 53% rename from Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php rename to Neos.ContentRepositoryRegistry/Classes/Processors/SubscriptionReplayProcessor.php index 7a5a1f9013f..83ce99eaca0 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Processors/ProjectionResetProcessor.php +++ b/Neos.ContentRepositoryRegistry/Classes/Processors/SubscriptionReplayProcessor.php @@ -3,22 +3,22 @@ namespace Neos\ContentRepositoryRegistry\Processors; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; -use Neos\ContentRepositoryRegistry\Service\ProjectionService; /** * @internal */ -final class ProjectionResetProcessor implements ProcessorInterface +final readonly class SubscriptionReplayProcessor implements ProcessorInterface { public function __construct( - private readonly ProjectionService $projectionService, + private ContentRepositoryMaintainer $contentRepositoryMaintainer, ) { } public function run(ProcessingContext $context): void { - $this->projectionService->resetAllProjections(); + $this->contentRepositoryMaintainer->replayAllSubscriptions(); } } diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationService.php b/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationService.php index 3348f1b73b3..e1b3b4dae8e 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationService.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationService.php @@ -28,6 +28,12 @@ use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateClassification; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\Engine\SubscriptionEngine; +use Neos\ContentRepository\Core\Subscription\Store\SubscriptionCriteria; +use Neos\ContentRepository\Core\Subscription\Store\SubscriptionStoreInterface; +use Neos\ContentRepository\Core\Subscription\Subscription; +use Neos\ContentRepository\Core\Subscription\SubscriptionId; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\ContentRepositoryRegistry\Command\MigrateEventsCommandController; use Neos\ContentRepositoryRegistry\Factory\EventStore\DoctrineEventStoreFactory; use Neos\EventStore\EventStoreInterface; @@ -44,6 +50,7 @@ use Neos\Neos\Domain\Model\WorkspaceClassification; use Neos\Neos\Domain\Model\WorkspaceRole; use Neos\Neos\Domain\Model\WorkspaceRoleSubjectType; +use Doctrine\DBAL\Exception as DBALException; /** * Content Repository service to perform migrations of events. @@ -59,6 +66,7 @@ final class EventMigrationService implements ContentRepositoryServiceInterface private array $eventsModified = []; public function __construct( + private readonly SubscriptionEngine $subscriptionEngine, private readonly ContentRepositoryId $contentRepositoryId, private readonly EventStoreInterface $eventStore, private readonly Connection $connection @@ -922,6 +930,59 @@ public function copyNodesStatus(\Closure $outputFn): void $outputFn('NOTE: To reduce the number of matched content streams and to cleanup the event store run `./flow contentStream:removeDangling` and `./flow contentStream:pruneRemovedFromEventStream`'); } + public function migrateCheckpointsToSubscriptions(\Closure $outputFn): void + { + /** @var SubscriptionStoreInterface $subscriptionStore */ + $subscriptionStore = (new \ReflectionClass($this->subscriptionEngine))->getProperty('subscriptionStore')->getValue($this->subscriptionEngine); + $subscriptionStore->setup(); + + foreach ([ + 'contentGraph' => 'cr_%s_p_graph_checkpoint', + 'Neos.Neos:DocumentUriPathProjection' => 'cr_%s_p_neos_documenturipath_checkpoint', + 'Neos.Neos:PendingChangesProjection' => 'cr_%s_p_neos_change_checkpoint', + ] as $subscriberId => $projectionCheckpointTablePattern) { + $projectionCheckpointTable = sprintf($projectionCheckpointTablePattern, $this->contentRepositoryId->value); + try { + $rows = $this->connection->fetchAllAssociative("SELECT appliedsequencenumber from {$projectionCheckpointTable}"); + } catch (DBALException $e) { + $outputFn(sprintf('Could not migrate subscriber %s, please replay: %s', $subscriberId, $e->getMessage())); + continue; + } + $first = reset($rows); + if (count($rows) !== 1 || !isset($first['appliedsequencenumber'])) { + $outputFn(sprintf('Could not migrate subscriber %s, please replay. Expected exactly one appliedsequencenumber', $subscriberId)); + continue; + } + + $subscription = $subscriptionStore->findByCriteriaForUpdate(SubscriptionCriteria::create([SubscriptionId::fromString($subscriberId)]))->first(); + + if ($subscription === null) { + $subscriptionStore->add( + new Subscription( + SubscriptionId::fromString($subscriberId), + SubscriptionStatus::ACTIVE, + SequenceNumber::fromInteger((int)$first['appliedsequencenumber']), + null, + null + ) + ); + $outputFn(sprintf('Added subscriber %s with active and position %s', $subscriberId, $first['appliedsequencenumber'])); + } else { + if ($subscription->status === SubscriptionStatus::ACTIVE) { + $outputFn(sprintf('Subscriber %s is already active', $subscriberId)); + continue; + } + $subscriptionStore->update( + SubscriptionId::fromString($subscriberId), + SubscriptionStatus::ACTIVE, + SequenceNumber::fromInteger((int)$first['appliedsequencenumber']), + null + ); + $outputFn(sprintf('Updated subscriber %s to active and position %s', $subscriberId, $first['appliedsequencenumber'])); + } + } + } + /** ------------------------ */ /** diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationServiceFactory.php b/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationServiceFactory.php index f61935dccaa..2e2dcea0f3f 100644 --- a/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationServiceFactory.php +++ b/Neos.ContentRepositoryRegistry/Classes/Service/EventMigrationServiceFactory.php @@ -32,6 +32,7 @@ public function build(ContentRepositoryServiceFactoryDependencies $serviceFactor } return new EventMigrationService( + $serviceFactoryDependencies->subscriptionEngine, $serviceFactoryDependencies->contentRepositoryId, $serviceFactoryDependencies->eventStore, $this->connection diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php deleted file mode 100644 index 06f11984d74..00000000000 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php +++ /dev/null @@ -1,125 +0,0 @@ -resolveProjectionClassName($projectionAliasOrClassName); - $this->contentRepository->resetProjectionState($projectionClassName); - $this->contentRepository->catchUpProjection($projectionClassName, $options); - } - - public function replayAllProjections(CatchUpOptions $options, ?\Closure $progressCallback = null): void - { - foreach ($this->projectionClassNamesAndAliases() as $classNamesAndAlias) { - if ($progressCallback) { - $progressCallback($classNamesAndAlias['alias']); - } - $this->contentRepository->resetProjectionState($classNamesAndAlias['className']); - $this->contentRepository->catchUpProjection($classNamesAndAlias['className'], $options); - } - } - - public function resetAllProjections(): void - { - foreach ($this->projectionClassNamesAndAliases() as $classNamesAndAlias) { - $this->contentRepository->resetProjectionState($classNamesAndAlias['className']); - } - } - - public function catchupProjection(string $projectionAliasOrClassName, CatchUpOptions $options): void - { - $projectionClassName = $this->resolveProjectionClassName($projectionAliasOrClassName); - $this->contentRepository->catchUpProjection($projectionClassName, $options); - } - - public function catchupAllProjections(CatchUpOptions $options, ?\Closure $progressCallback = null): void - { - foreach ($this->projectionClassNamesAndAliases() as $classNamesAndAlias) { - if ($progressCallback) { - $progressCallback($classNamesAndAlias['alias']); - } - $this->contentRepository->catchUpProjection($classNamesAndAlias['className'], $options); - } - } - - public function highestSequenceNumber(): SequenceNumber - { - foreach ($this->eventStore->load(VirtualStreamName::all())->backwards()->limit(1) as $eventEnvelope) { - return $eventEnvelope->sequenceNumber; - } - return SequenceNumber::none(); - } - - public function numberOfProjections(): int - { - return count($this->projections); - } - - /** - * @return class-string> - */ - private function resolveProjectionClassName(string $projectionAliasOrClassName): string - { - $lowerCaseProjectionName = strtolower($projectionAliasOrClassName); - $projectionClassNamesAndAliases = $this->projectionClassNamesAndAliases(); - foreach ($projectionClassNamesAndAliases as $classNamesAndAlias) { - if (strtolower($classNamesAndAlias['className']) === $lowerCaseProjectionName || strtolower($classNamesAndAlias['alias']) === $lowerCaseProjectionName) { - return $classNamesAndAlias['className']; - } - } - throw new \InvalidArgumentException(sprintf( - 'The projection "%s" is not registered for this Content Repository. The following projection aliases (or fully qualified class names) can be used: %s', - $projectionAliasOrClassName, - implode('', array_map(static fn (array $classNamesAndAlias) => sprintf(chr(10) . ' * %s (%s)', $classNamesAndAlias['alias'], $classNamesAndAlias['className']), $projectionClassNamesAndAliases)) - ), 1680519624); - } - - /** - * @return array>, alias: string}> - */ - private function projectionClassNamesAndAliases(): array - { - return array_map( - static fn (string $projectionClassName) => [ - 'className' => $projectionClassName, - 'alias' => self::projectionAlias($projectionClassName), - ], - $this->projections->getClassNames() - ); - } - - private static function projectionAlias(string $className): string - { - $alias = lcfirst(substr(strrchr($className, '\\') ?: '\\' . $className, 1)); - if (str_ends_with($alias, 'Projection')) { - $alias = substr($alias, 0, -10); - } - return $alias; - } -} diff --git a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php b/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php deleted file mode 100644 index 92114d47f1a..00000000000 --- a/Neos.ContentRepositoryRegistry/Classes/Service/ProjectionServiceFactory.php +++ /dev/null @@ -1,28 +0,0 @@ - - * @internal - */ -#[Flow\Scope("singleton")] -final class ProjectionServiceFactory implements ContentRepositoryServiceFactoryInterface -{ - public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface - { - return new ProjectionService( - $serviceFactoryDependencies->projectionsAndCatchUpHooks->projections, - $serviceFactoryDependencies->contentRepository, - $serviceFactoryDependencies->eventStore, - ); - } -} diff --git a/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHook.php b/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHook.php index febdd096d20..3644ecfa414 100644 --- a/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHook.php +++ b/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHook.php @@ -4,9 +4,10 @@ namespace Neos\ContentRepositoryRegistry\SubgraphCachingInMemory; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\Flow\Annotations as Flow; use Neos\ContentRepository\Core\EventStore\EventInterface; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\EventStore\Model\EventEnvelope; /** @@ -23,7 +24,7 @@ public function __construct(private readonly SubgraphCachePool $subgraphCachePoo { } - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { } @@ -36,7 +37,7 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event $this->subgraphCachePool->reset(); } - public function onBeforeBatchCompleted(): void + public function onAfterBatchCompleted(): void { } diff --git a/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHookFactory.php b/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHookFactory.php index cebe03ebb43..f3017920113 100644 --- a/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHookFactory.php +++ b/Neos.ContentRepositoryRegistry/Classes/SubgraphCachingInMemory/FlushSubgraphCachePoolCatchUpHookFactory.php @@ -4,9 +4,9 @@ namespace Neos\ContentRepositoryRegistry\SubgraphCachingInMemory; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; /** diff --git a/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml b/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml index df582d41eba..70297ff0415 100644 --- a/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml +++ b/Neos.ContentRepositoryRegistry/Configuration/Objects.yaml @@ -1,9 +1,3 @@ -Neos\ContentRepositoryRegistry\ContentRepositoryRegistry: - arguments: - 1: - setting: Neos.ContentRepositoryRegistry - -# !!! UGLY WORKAROUNDS, because we cannot wire non-Flow class constructor arguments here. # This adds a soft-dependency to the neos/contentgraph-doctrinedbaladapter package Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory: @@ -14,3 +8,12 @@ Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory: value: 'Neos\ContentGraph\DoctrineDbalAdapter\DoctrineDbalContentGraphProjectionFactory' 2: object: 'Doctrine\DBAL\Connection' + +'Neos.ContentRepositoryRegistry:Logger': + className: Psr\Log\LoggerInterface + scope: singleton + factoryObjectName: Neos\Flow\Log\PsrLoggerFactoryInterface + factoryMethodName: get + arguments: + 1: + value: contentRepositoryLogger diff --git a/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml b/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml index 80574bfb60f..d78307056a8 100644 --- a/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml +++ b/Neos.ContentRepositoryRegistry/Configuration/Settings.yaml @@ -12,6 +12,21 @@ Neos: ignoredClasses: Neos\\ContentRepository\\SharedModel\\NodeType\\NodeTypeManager: true + log: + psr3: + 'Neos\Flow\Log\PsrLoggerFactory': + contentRepositoryLogger: + default: + class: Neos\Flow\Log\Backend\FileBackend + options: + # todo context aware? FLOW_APPLICATION_CONTEXT .. but that contains / + logFileURL: '%FLOW_PATH_DATA%Logs/ContentRepository.log' + createParentDirectories: true + severityThreshold: '%LOG_INFO%' + maximumLogFileSize: 10485760 + logFilesToKeep: 1 + logMessageOrigin: false + ContentRepositoryRegistry: contentRepositories: default: @@ -37,6 +52,9 @@ Neos: clock: factoryObjectName: Neos\ContentRepositoryRegistry\Factory\Clock\SystemClockFactory + subscriptionStore: + factoryObjectName: Neos\ContentRepositoryRegistry\Factory\SubscriptionStore\SubscriptionStoreFactory + propertyConverters: DateTimeNormalizer: className: Symfony\Component\Serializer\Normalizer\DateTimeNormalizer diff --git a/Neos.ContentRepositoryRegistry/Tests/Functional/ContentRepositoryMaintenanceCommandControllerTest.php b/Neos.ContentRepositoryRegistry/Tests/Functional/ContentRepositoryMaintenanceCommandControllerTest.php new file mode 100644 index 00000000000..7dd43195d58 --- /dev/null +++ b/Neos.ContentRepositoryRegistry/Tests/Functional/ContentRepositoryMaintenanceCommandControllerTest.php @@ -0,0 +1,142 @@ +crController = $this->getObject(CrCommandController::class); + $this->subscriptionController = $this->getObject(SubscriptionCommandController::class); + + $this->response = new Response(); + $this->bufferedOutput = new BufferedOutput(); + + ObjectAccess::setProperty($this->crController, 'response', $this->response, true); + ObjectAccess::getProperty($this->crController, 'output', true)->setOutput($this->bufferedOutput); + + ObjectAccess::setProperty($this->subscriptionController, 'response', $this->response, true); + ObjectAccess::getProperty($this->subscriptionController, 'output', true)->setOutput($this->bufferedOutput); + } + + /** @test */ + public function setupOnEmptyEventStore(): void + { + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->crController->setupCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + + // projections are marked active because the event store is empty + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::none()); + + $this->crController->statusCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + } + + /** @test */ + public function setupOnModifiedEventStore(): void + { + $this->eventStore->setup(); + $this->commitExampleContentStreamEvent(); + + $this->fakeProjection->expects(self::once())->method('setUp'); + $this->fakeProjection->expects(self::once())->method('apply'); + $this->fakeProjection->expects(self::once())->method('resetState'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->crController->setupCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $this->crController->statusCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + + $this->subscriptionController->replayCommand(subscription: 'contentGraph', contentRepository: $this->contentRepository->id->value, force: true, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::BOOTING, SequenceNumber::none()); + + $this->subscriptionController->replayAllCommand(contentRepository: $this->contentRepository->id->value, force: true, quiet: true); + + $this->expectOkayStatus('contentGraph', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:FakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(1)); + } + + /** @test */ + public function projectionInError(): void + { + $this->eventStore->setup(); + $this->fakeProjection->expects(self::any())->method('setUp'); + $this->fakeProjection->expects(self::any())->method('apply'); + $this->fakeProjection->expects(self::any())->method('status')->willReturn(ProjectionStatus::ok()); + + $this->crController->setupCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + self::assertEmpty($this->bufferedOutput->fetch()); + + $this->secondFakeProjection->injectSaboteur(fn () => throw new \RuntimeException('This projection is kaputt.')); + + try { + $this->contentRepository->handle(CreateRootWorkspace::create( + WorkspaceName::forLive(), + ContentStreamId::create() + )); + } catch (\RuntimeException) { + } + + self::assertEquals( + SubscriptionStatus::ERROR, + $this->subscriptionStatus('Vendor.Package:SecondFakeProjection')?->subscriptionStatus + ); + + try { + $this->crController->statusCommand(contentRepository: $this->contentRepository->id->value, quiet: true); + } catch (StopCommandException) { + } + // exit error code because one projection has a failure + self::assertEquals(1, $this->response->getExitCode()); + self::assertEmpty($this->bufferedOutput->fetch()); + + // repair projection + $this->secondFakeProjection->killSaboteur(); + $this->subscriptionController->replayCommand(subscription: 'Vendor.Package:SecondFakeProjection', contentRepository: $this->contentRepository->id->value, force: true, quiet: true); + + $this->expectOkayStatus('Vendor.Package:SecondFakeProjection', SubscriptionStatus::ACTIVE, SequenceNumber::fromInteger(2)); + } +} diff --git a/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php b/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php index 631175e83d8..6c9c7ab8c81 100644 --- a/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php +++ b/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHook.php @@ -16,7 +16,7 @@ use Neos\ContentRepository\Core\Feature\NodeVariation\Event\NodePeerVariantWasCreated; use Neos\ContentRepository\Core\Feature\NodeVariation\Event\NodeSpecializationVariantWasCreated; use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasDiscarded; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\Filter\FindDescendantNodesFilter; use Neos\ContentRepository\Core\Projection\ContentGraph\Node; @@ -25,6 +25,7 @@ use Neos\ContentRepository\Core\SharedModel\Exception\WorkspaceDoesNotExist; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; use Neos\Neos\AssetUsage\Service\AssetUsageIndexingService; @@ -40,7 +41,7 @@ public function __construct( ) { } - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { } @@ -84,8 +85,7 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event }; } - - public function onBeforeBatchCompleted(): void + public function onAfterBatchCompleted(): void { } diff --git a/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHookFactory.php b/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHookFactory.php index 4188a10072f..71cf34fe2ea 100644 --- a/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHookFactory.php +++ b/Neos.Neos/Classes/AssetUsage/CatchUpHook/AssetUsageCatchUpHookFactory.php @@ -14,8 +14,8 @@ * source code. */ -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\Neos\AssetUsage\Service\AssetUsageIndexingService; diff --git a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php index 0b94195c9d1..9cb7cf60b24 100644 --- a/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php +++ b/Neos.Neos/Classes/Domain/Pruning/ContentRepositoryPruningProcessor.php @@ -14,22 +14,25 @@ namespace Neos\Neos\Domain\Pruning; -use Neos\ContentRepository\Core\Service\ContentStreamPruner; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; /** - * Pruning processor that removes all events from the given cr + * Pruning processor that removes all events from the given cr and resets the projections */ final readonly class ContentRepositoryPruningProcessor implements ProcessorInterface { public function __construct( - private ContentStreamPruner $contentStreamPruner, + private ContentRepositoryMaintainer $contentRepositoryMaintainer, ) { } public function run(ProcessingContext $context): void { - $this->contentStreamPruner->pruneAllWorkspacesAndContentStreamsFromEventStream(); + $result = $this->contentRepositoryMaintainer->prune(); + if ($result !== null) { + throw new \RuntimeException($result->getMessage(), 1732461335); + } } } diff --git a/Neos.Neos/Classes/Domain/Service/SiteImportService.php b/Neos.Neos/Classes/Domain/Service/SiteImportService.php index 741424a02e2..ebe76134d19 100644 --- a/Neos.Neos/Classes/Domain/Service/SiteImportService.php +++ b/Neos.Neos/Classes/Domain/Service/SiteImportService.php @@ -17,9 +17,12 @@ use Doctrine\DBAL\Exception as DBALException; use League\Flysystem\Filesystem; use League\Flysystem\Local\LocalFilesystemAdapter; -use Neos\ContentRepository\Core\ContentRepository; +use Neos\ContentRepository\Core\Projection\ProjectionStatusType; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainer; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\ProjectionSubscriptionStatus; use Neos\ContentRepository\Export\Factory\EventStoreImportProcessorFactory; use Neos\ContentRepository\Export\ProcessingContext; use Neos\ContentRepository\Export\ProcessorInterface; @@ -27,8 +30,8 @@ use Neos\ContentRepository\Export\Processors\AssetRepositoryImportProcessor; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Processors\ProjectionCatchupProcessor; -use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; +use Neos\ContentRepositoryRegistry\Processors\SubscriptionReplayProcessor; +use Neos\EventStore\Model\EventStore\StatusType; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\Doctrine\Service as DoctrineService; use Neos\Flow\Persistence\PersistenceManagerInterface; @@ -67,8 +70,10 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string } $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); + $contentRepositoryMaintainer = $this->contentRepositoryRegistry->buildService($contentRepositoryId, new ContentRepositoryMaintainerFactory()); + $this->requireDataBaseSchemaToBeSetup(); - $this->requireContentRepositoryToBeSetup($contentRepository); + $this->requireContentRepositoryToBeSetup($contentRepositoryMaintainer, $contentRepositoryId); $filesystem = new Filesystem(new LocalFilesystemAdapter($path)); $context = new ProcessingContext($filesystem, $onMessage); @@ -78,7 +83,9 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string 'Create Neos sites' => new SiteCreationProcessor($this->siteRepository, $this->domainRepository, $this->persistenceManager), 'Import events' => $this->contentRepositoryRegistry->buildService($contentRepositoryId, new EventStoreImportProcessorFactory(WorkspaceName::forLive(), keepEventIds: true)), 'Import assets' => new AssetRepositoryImportProcessor($this->assetRepository, $this->resourceRepository, $this->resourceManager, $this->persistenceManager), - 'Catchup all projections' => new ProjectionCatchupProcessor($this->contentRepositoryRegistry->buildService($contentRepositoryId, new ProjectionServiceFactory())), + // WARNING! We do a replay here even though it will redo the live workspace creation. But otherwise the catchup hooks cannot determine that they need to be skipped as it seems like a regular catchup + // In case we allow to import events into other root workspaces, or don't expect live to be empty (see Import events), this would need to be adjusted, as otherwise existing data will be replayed + 'Replay all subscriptions' => new SubscriptionReplayProcessor($contentRepositoryMaintainer), ]); foreach ($processors as $processorLabel => $processor) { @@ -87,11 +94,18 @@ public function importFromPath(ContentRepositoryId $contentRepositoryId, string } } - private function requireContentRepositoryToBeSetup(ContentRepository $contentRepository): void + private function requireContentRepositoryToBeSetup(ContentRepositoryMaintainer $contentRepositoryMaintainer, ContentRepositoryId $contentRepositoryId): void { - $status = $contentRepository->status(); - if (!$status->isOk()) { - throw new \RuntimeException(sprintf('Content repository %s is not setup correctly, please run `./flow cr:setup`', $contentRepository->id->value)); + $status = $contentRepositoryMaintainer->status(); + if ($status->eventStoreStatus->type !== StatusType::OK) { + throw new \RuntimeException(sprintf('Content repository %s is not setup correctly, please run `./flow cr:setup`', $contentRepositoryId->value)); + } + foreach ($status->subscriptionStatus as $status) { + if ($status instanceof ProjectionSubscriptionStatus) { + if ($status->setupStatus->type !== ProjectionStatusType::OK) { + throw new \RuntimeException(sprintf('Projection %s in content repository %s is not setup correctly, please run `./flow cr:setup`', $status->subscriptionId->value, $contentRepositoryId->value)); + } + } } } diff --git a/Neos.Neos/Classes/Domain/Service/SitePruningService.php b/Neos.Neos/Classes/Domain/Service/SitePruningService.php index 9a1c7c67537..18ab6f1d78f 100644 --- a/Neos.Neos/Classes/Domain/Service/SitePruningService.php +++ b/Neos.Neos/Classes/Domain/Service/SitePruningService.php @@ -16,7 +16,7 @@ use League\Flysystem\Filesystem; use League\Flysystem\Local\LocalFilesystemAdapter; -use Neos\ContentRepository\Core\Service\ContentStreamPrunerFactory; +use Neos\ContentRepository\Core\Service\ContentRepositoryMaintainerFactory; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\ContentRepository\Export\ProcessingContext; @@ -24,8 +24,6 @@ use Neos\ContentRepository\Export\Processors; use Neos\ContentRepository\Export\Severity; use Neos\ContentRepositoryRegistry\ContentRepositoryRegistry; -use Neos\ContentRepositoryRegistry\Processors\ProjectionResetProcessor; -use Neos\ContentRepositoryRegistry\Service\ProjectionServiceFactory; use Neos\Flow\Annotations as Flow; use Neos\Flow\Persistence\PersistenceManagerInterface; use Neos\Neos\Domain\Pruning\ContentRepositoryPruningProcessor; @@ -66,17 +64,11 @@ public function pruneAll(ContentRepositoryId $contentRepositoryId, \Closure $onP $this->domainRepository, $this->persistenceManager ), - 'Prune content repository' => new ContentRepositoryPruningProcessor( - $this->contentRepositoryRegistry->buildService( - $contentRepositoryId, - new ContentStreamPrunerFactory() - ) - ), 'Prune roles and metadata' => new RoleAndMetadataPruningProcessor($contentRepositoryId, $this->workspaceMetadataAndRoleRepository), - 'Reset all projections' => new ProjectionResetProcessor( + 'Prune content repository' => new ContentRepositoryPruningProcessor( $this->contentRepositoryRegistry->buildService( $contentRepositoryId, - new ProjectionServiceFactory() + new ContentRepositoryMaintainerFactory() ) ) ]); diff --git a/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php b/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php index 1dc70d4fac7..b26f90ebef2 100644 --- a/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php +++ b/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHook.php @@ -10,8 +10,9 @@ use Neos\ContentRepository\Core\Feature\NodeMove\Event\NodeAggregateWasMoved; use Neos\ContentRepository\Core\Feature\NodeRemoval\Event\NodeAggregateWasRemoved; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; use Neos\Flow\Mvc\Routing\RouterCachingService; use Neos\Neos\FrontendRouting\Exception\NodeNotFoundException; @@ -32,7 +33,7 @@ public function __construct( ) { } - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { // Nothing to do here } @@ -59,7 +60,7 @@ public function onAfterEvent(EventInterface $eventInstance, EventEnvelope $event }; } - public function onBeforeBatchCompleted(): void + public function onAfterBatchCompleted(): void { // Nothing to do here } diff --git a/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHookFactory.php b/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHookFactory.php index cbdd469d930..b0c4faf2dd6 100644 --- a/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHookFactory.php +++ b/Neos.Neos/Classes/FrontendRouting/CatchUpHook/RouterCacheHookFactory.php @@ -4,9 +4,9 @@ namespace Neos\Neos\FrontendRouting\CatchUpHook; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\Flow\Mvc\Routing\RouterCachingService; use Neos\Neos\FrontendRouting\Projection\DocumentUriPathFinder; diff --git a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php index de859059ae0..016814020bc 100644 --- a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php +++ b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjection.php @@ -24,16 +24,13 @@ use Neos\ContentRepository\Core\Feature\RootNodeCreation\Event\RootNodeAggregateWithNodeWasCreated; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; -use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\NodeType\NodeTypeManager; use Neos\ContentRepository\Core\NodeType\NodeTypeName; -use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\Projection\WithMarkStaleInterface; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; -use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; use Neos\Neos\Domain\Model\SiteNodeName; use Neos\Neos\FrontendRouting\Exception\NodeNotFoundException; @@ -47,7 +44,6 @@ final class DocumentUriPathProjection implements ProjectionInterface, WithMarkSt 'shortcutTarget' => Types::JSON, ]; - private DbalCheckpointStorage $checkpointStorage; private ?DocumentUriPathFinder $stateAccessor = null; /** @@ -60,11 +56,6 @@ public function __construct( private readonly Connection $dbal, private readonly string $tableNamePrefix, ) { - $this->checkpointStorage = new DbalCheckpointStorage( - $this->dbal, - $this->tableNamePrefix . '_checkpoint', - self::class - ); } public function setUp(): void @@ -72,18 +63,10 @@ public function setUp(): void foreach ($this->determineRequiredSqlStatements() as $statement) { $this->dbal->executeStatement($statement); } - $this->checkpointStorage->setUp(); } public function status(): ProjectionStatus { - $checkpointStorageStatus = $this->checkpointStorage->status(); - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { - return ProjectionStatus::error($checkpointStorageStatus->details); - } - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { - return ProjectionStatus::setupRequired($checkpointStorageStatus->details); - } try { $this->dbal->connect(); } catch (\Throwable $e) { @@ -113,11 +96,9 @@ private function determineRequiredSqlStatements(): array } - public function reset(): void + public function resetState(): void { $this->truncateDatabaseTables(); - $this->checkpointStorage->acquireLock(); - $this->checkpointStorage->updateAndReleaseLock(SequenceNumber::none()); $this->stateAccessor = null; } @@ -130,27 +111,6 @@ private function truncateDatabaseTables(): void } } - - public function canHandle(EventInterface $event): bool - { - return in_array($event::class, [ - RootNodeAggregateWithNodeWasCreated::class, - RootNodeAggregateDimensionsWereUpdated::class, - NodeAggregateWithNodeWasCreated::class, - NodeAggregateTypeWasChanged::class, - NodePeerVariantWasCreated::class, - NodeGeneralizationVariantWasCreated::class, - NodeSpecializationVariantWasCreated::class, - SubtreeWasTagged::class, - SubtreeWasUntagged::class, - NodeAggregateWasRemoved::class, - NodePropertiesWereSet::class, - NodeAggregateWasMoved::class, - DimensionSpacePointWasMoved::class, - DimensionShineThroughWasAdded::class, - ]); - } - public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void { match ($event::class) { @@ -168,15 +128,10 @@ public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void NodeAggregateWasMoved::class => $this->whenNodeAggregateWasMoved($event), DimensionSpacePointWasMoved::class => $this->whenDimensionSpacePointWasMoved($event), DimensionShineThroughWasAdded::class => $this->whenDimensionShineThroughWasAdded($event), - default => throw new \InvalidArgumentException(sprintf('Unsupported event %s', get_debug_type($event))), + default => null, }; } - public function getCheckpointStorage(): DbalCheckpointStorage - { - return $this->checkpointStorage; - } - public function getState(): DocumentUriPathFinder { if (!$this->stateAccessor) { diff --git a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjectionFactory.php b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjectionFactory.php index 570e0f5485d..c70dfb2dbd3 100644 --- a/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjectionFactory.php +++ b/Neos.Neos/Classes/FrontendRouting/Projection/DocumentUriPathProjectionFactory.php @@ -5,7 +5,7 @@ namespace Neos\Neos\FrontendRouting\Projection; use Doctrine\DBAL\Connection; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; @@ -35,7 +35,7 @@ public static function projectionTableNamePrefix( public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, + SubscriberFactoryDependencies $projectionFactoryDependencies, array $options, ): DocumentUriPathProjection { diff --git a/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php b/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php index ec1b8ac4c5c..74ecea59ce0 100644 --- a/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php +++ b/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushing.php @@ -33,13 +33,14 @@ use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; use Neos\ContentRepository\Core\Feature\WorkspacePublication\Event\WorkspaceWasDiscarded; use Neos\ContentRepository\Core\Feature\WorkspaceRebase\Event\WorkspaceWasRebased; -use Neos\ContentRepository\Core\Projection\CatchUpHookInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\NodeAggregate; use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId; use Neos\ContentRepository\Core\SharedModel\Exception\WorkspaceDoesNotExist; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateIds; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; +use Neos\ContentRepository\Core\Subscription\SubscriptionStatus; use Neos\EventStore\Model\EventEnvelope; /** @@ -133,7 +134,7 @@ public function canHandle(EventInterface $event): bool ]); } - public function onBeforeCatchUp(): void + public function onBeforeCatchUp(SubscriptionStatus $subscriptionStatus): void { } @@ -241,7 +242,7 @@ private function scheduleCacheFlushJobForWorkspaceName( ); } - public function onBeforeBatchCompleted(): void + public function onAfterBatchCompleted(): void { } diff --git a/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushingFactory.php b/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushingFactory.php index a988f1bcac6..35a07b11b70 100644 --- a/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushingFactory.php +++ b/Neos.Neos/Classes/Fusion/Cache/GraphProjectorCatchUpHookForCacheFlushingFactory.php @@ -14,8 +14,8 @@ * source code. */ -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryDependencies; -use Neos\ContentRepository\Core\Projection\CatchUpHookFactoryInterface; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryDependencies; +use Neos\ContentRepository\Core\Projection\CatchUpHook\CatchUpHookFactoryInterface; use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; /** diff --git a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php index 34613131d9b..fa4de8269ef 100644 --- a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php +++ b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjection.php @@ -38,15 +38,12 @@ use Neos\ContentRepository\Core\Feature\NodeVariation\Event\NodeSpecializationVariantWasCreated; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasTagged; use Neos\ContentRepository\Core\Feature\SubtreeTagging\Event\SubtreeWasUntagged; -use Neos\ContentRepository\Core\Infrastructure\DbalCheckpointStorage; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaDiff; use Neos\ContentRepository\Core\Infrastructure\DbalSchemaFactory; -use Neos\ContentRepository\Core\Projection\CheckpointStorageStatusType; use Neos\ContentRepository\Core\Projection\ProjectionInterface; use Neos\ContentRepository\Core\Projection\ProjectionStatus; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId; -use Neos\EventStore\Model\Event\SequenceNumber; use Neos\EventStore\Model\EventEnvelope; /** @@ -62,17 +59,10 @@ class ChangeProjection implements ProjectionInterface */ private ?ChangeFinder $changeFinder = null; - private DbalCheckpointStorage $checkpointStorage; - public function __construct( private readonly Connection $dbal, private readonly string $tableNamePrefix, ) { - $this->checkpointStorage = new DbalCheckpointStorage( - $this->dbal, - $this->tableNamePrefix . '_checkpoint', - self::class - ); } /** @@ -84,18 +74,10 @@ public function setUp(): void foreach ($this->determineRequiredSqlStatements() as $statement) { $this->dbal->executeStatement($statement); } - $this->checkpointStorage->setUp(); } public function status(): ProjectionStatus { - $checkpointStorageStatus = $this->checkpointStorage->status(); - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::ERROR) { - return ProjectionStatus::error($checkpointStorageStatus->details); - } - if ($checkpointStorageStatus->type === CheckpointStorageStatusType::SETUP_REQUIRED) { - return ProjectionStatus::setupRequired($checkpointStorageStatus->details); - } try { $this->dbal->connect(); } catch (\Throwable $e) { @@ -147,31 +129,9 @@ private function determineRequiredSqlStatements(): array return $statements; } - public function reset(): void + public function resetState(): void { $this->dbal->exec('TRUNCATE ' . $this->tableNamePrefix); - $this->checkpointStorage->acquireLock(); - $this->checkpointStorage->updateAndReleaseLock(SequenceNumber::none()); - } - - public function canHandle(EventInterface $event): bool - { - return in_array($event::class, [ - NodeAggregateWasMoved::class, - NodePropertiesWereSet::class, - NodeReferencesWereSet::class, - NodeAggregateWithNodeWasCreated::class, - SubtreeWasTagged::class, - SubtreeWasUntagged::class, - NodeAggregateWasRemoved::class, - DimensionSpacePointWasMoved::class, - NodeGeneralizationVariantWasCreated::class, - NodeSpecializationVariantWasCreated::class, - NodePeerVariantWasCreated::class, - NodeAggregateTypeWasChanged::class, - NodeAggregateNameWasChanged::class, - ContentStreamWasRemoved::class, - ]); } public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void @@ -191,15 +151,10 @@ public function apply(EventInterface $event, EventEnvelope $eventEnvelope): void NodeAggregateTypeWasChanged::class => $this->whenNodeAggregateTypeWasChanged($event), NodeAggregateNameWasChanged::class => $this->whenNodeAggregateNameWasChanged($event), ContentStreamWasRemoved::class => $this->whenContentStreamWasRemoved($event), - default => throw new \InvalidArgumentException(sprintf('Unsupported event %s', get_debug_type($event))), + default => null, }; } - public function getCheckpointStorage(): DbalCheckpointStorage - { - return $this->checkpointStorage; - } - public function getState(): ChangeFinder { if (!$this->changeFinder) { diff --git a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjectionFactory.php b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjectionFactory.php index 8b231897d78..244c1352426 100644 --- a/Neos.Neos/Classes/PendingChangesProjection/ChangeProjectionFactory.php +++ b/Neos.Neos/Classes/PendingChangesProjection/ChangeProjectionFactory.php @@ -15,7 +15,7 @@ namespace Neos\Neos\PendingChangesProjection; use Doctrine\DBAL\Connection; -use Neos\ContentRepository\Core\Factory\ProjectionFactoryDependencies; +use Neos\ContentRepository\Core\Factory\SubscriberFactoryDependencies; use Neos\ContentRepository\Core\Projection\ProjectionFactoryInterface; /** @@ -29,7 +29,7 @@ public function __construct( } public function build( - ProjectionFactoryDependencies $projectionFactoryDependencies, + SubscriberFactoryDependencies $projectionFactoryDependencies, array $options, ): ChangeProjection { return new ChangeProjection( diff --git a/Neos.Neos/Tests/Behavior/Features/Bootstrap/ContentRepositorySecurityTrait.php b/Neos.Neos/Tests/Behavior/Features/Bootstrap/ContentRepositorySecurityTrait.php index 8d22c21adfc..5b0a2d46eb7 100644 --- a/Neos.Neos/Tests/Behavior/Features/Bootstrap/ContentRepositorySecurityTrait.php +++ b/Neos.Neos/Tests/Behavior/Features/Bootstrap/ContentRepositorySecurityTrait.php @@ -17,7 +17,7 @@ use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryDependencies; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface; use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface; -use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphProjectionInterface; +use Neos\ContentRepository\Core\Projection\ContentGraph\ContentGraphReadModelInterface; use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId; use Neos\ContentRepository\Core\SharedModel\Workspace\WorkspaceName; use Neos\Flow\Mvc\ActionRequest; @@ -67,19 +67,19 @@ private function enableContentRepositorySecurity(): void return; } $contentRepositoryAuthProviderFactory = $this->getObject(ContentRepositoryAuthProviderFactory::class); - $contentGraphProjection = $this->getContentRepositoryService(new class implements ContentRepositoryServiceFactoryInterface { + $contentGraphReadModel = $this->getContentRepositoryService(new class implements ContentRepositoryServiceFactoryInterface { public function build(ContentRepositoryServiceFactoryDependencies $serviceFactoryDependencies): ContentRepositoryServiceInterface { - $contentGraphProjection = $serviceFactoryDependencies->projectionsAndCatchUpHooks->contentGraphProjection; - return new class ($contentGraphProjection) implements ContentRepositoryServiceInterface { + $contentGraphReadModel = $serviceFactoryDependencies->contentGraphReadModel; + return new class ($contentGraphReadModel) implements ContentRepositoryServiceInterface { public function __construct( - public ContentGraphProjectionInterface $contentGraphProjection, + public ContentGraphReadModelInterface $contentGraphReadModel, ) { } }; } - })->contentGraphProjection; - $contentRepositoryAuthProvider = $contentRepositoryAuthProviderFactory->build($this->currentContentRepository->id, $contentGraphProjection->getState()); + })->contentGraphReadModel; + $contentRepositoryAuthProvider = $contentRepositoryAuthProviderFactory->build($this->currentContentRepository->id, $contentGraphReadModel); FakeAuthProvider::replaceAuthProvider($contentRepositoryAuthProvider); $this->crSecurity_contentRepositorySecurityEnabled = true; diff --git a/Neos.Neos/Tests/Behavior/Features/FrontendRouting/Basic.feature b/Neos.Neos/Tests/Behavior/Features/FrontendRouting/Basic.feature index 640ac7e62bf..c9fde1e6e6b 100644 --- a/Neos.Neos/Tests/Behavior/Features/FrontendRouting/Basic.feature +++ b/Neos.Neos/Tests/Behavior/Features/FrontendRouting/Basic.feature @@ -113,7 +113,7 @@ Feature: Basic routing functionality (match & resolve document nodes in one dime # !!! when caches were still enabled (without calling DocumentUriPathFinder->disableCache()), the replay below will # show really "interesting" (non-correct) results. This was bug #4253. - When I replay the "Neos\Neos\FrontendRouting\Projection\DocumentUriPathProjection" projection + When I replay the "Neos.Neos:DocumentUriPathProjection" projection Then the node "sir-david-nodenborough" in content stream "cs-identifier" and dimension "{}" should resolve to URL "/david-nodenborough-updated-b" And the node "earl-o-documentbourgh" in content stream "cs-identifier" and dimension "{}" should resolve to URL "/david-nodenborough-updated-b/earl-document" diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index 12d96178fbb..2477370866c 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -1,9 +1,19 @@ parameters: ignoreErrors: - - message: "#^The internal method \"Neos\\\\ContentRepository\\\\Core\\\\Projection\\\\Projections\\:\\:getClassNames\" is called\\.$#" + message: "#^The internal method \"Neos\\\\ContentRepository\\\\Core\\\\Subscription\\\\SubscriptionIds\\:\\:toStringArray\" is called\\.$#" count: 1 - path: Neos.ContentRepositoryRegistry/Classes/Service/ProjectionService.php + path: Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php + + - + message: "#^The internal method \"Neos\\\\ContentRepository\\\\Core\\\\Subscription\\\\SubscriptionStatusFilter\\:\\:isEmpty\" is called\\.$#" + count: 1 + path: Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php + + - + message: "#^The internal method \"Neos\\\\ContentRepository\\\\Core\\\\Subscription\\\\SubscriptionStatusFilter\\:\\:toStringArray\" is called\\.$#" + count: 1 + path: Neos.ContentRepositoryRegistry/Classes/Factory/SubscriptionStore/DoctrineSubscriptionStore.php - message: "#^Method Neos\\\\Neos\\\\Controller\\\\Backend\\\\MenuHelper\\:\\:buildModuleList\\(\\) return type has no value type specified in iterable type array\\.$#"