EZP-26686: Script to migrate legacy XML format to new ezmatrix value (#…

…11)

* EZP-26686: Script to migrate legacy XML format to new ezmatrix value

* fix: Code Review

src/bundle/Command/MigrateLegacyMatrixCommand.php

@@ -0,0 +1,342 @@
+<?php
+
+/**
+ * @copyright Copyright (C) eZ Systems AS. All rights reserved.
+ * @license For full copyright and license information view LICENSE file distributed with this source code.
+ */
+declare(strict_types=1);
+
+namespace EzSystems\EzPlatformMatrixFieldtypeBundle\Command;
+
+use Doctrine\DBAL\Connection;
+use Exception;
+use eZ\Publish\Core\Persistence\Legacy\Content\StorageFieldDefinition;
+use eZ\Publish\Core\Persistence\Legacy\Content\StorageFieldValue;
+use eZ\Publish\SPI\Persistence\Content\FieldValue;
+use eZ\Publish\SPI\Persistence\Content\Type\FieldDefinition;
+use EzSystems\EzPlatformMatrixFieldtype\FieldType\Converter\MatrixConverter;
+use RuntimeException;
+use SimpleXMLElement;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Helper\ProgressBar;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
+
+class MigrateLegacyMatrixCommand extends Command
+{
+    private const DEFAULT_ITERATION_COUNT = 1000;
+
+    private const EZMATRIX_IDENTIFIER = 'ezmatrix';
+
+    private const CONFIRMATION_ANSWER = 'yes';
+
+    protected static $defaultName = 'ezplatform:migrate:legacy_matrix';
+
+    /** @var \Doctrine\DBAL\Connection */
+    private $connection;
+
+    /**
+     * @param \Doctrine\DBAL\Connection $connection
+     */
+    public function __construct(
+        Connection $connection
+    ) {
+        $this->connection = $connection;
+
+        parent::__construct();
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    protected function configure()
+    {
+        $this
+            ->addOption(
+                'iteration-count',
+                'c',
+                InputOption::VALUE_OPTIONAL,
+                'Number of matrix FieldType values fetched into memory and processed at once',
+                self::DEFAULT_ITERATION_COUNT
+            );
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function execute(InputInterface $input, OutputInterface $output)
+    {
+        $io = new SymfonyStyle($input, $output);
+
+        if (!$input->isInteractive()) {
+            throw new RuntimeException('Command cannot be run in non-interactive mode.');
+        }
+
+        $io->caution('Read carefully. This operation is irreversible. Make sure you are using correct database and have backup.');
+        $answer = $io->ask('Are you sure you want to start migration? (type "' . self::CONFIRMATION_ANSWER . '" to confirm)');
+
+        if ($answer !== self::CONFIRMATION_ANSWER) {
+            $io->comment('Canceled.');
+
+            exit();
+        }
+
+        $io->comment('Migrating legacy ezmatrix fieldtype');
+
+        $iterationCount = (int)$input->getOption('iteration-count');
+        $converter = new MatrixConverter();
+
+        $contentClassAttributes = $this->getContentClassAttributes();
+
+        libxml_use_internal_errors(true);
+
+        foreach ($contentClassAttributes as $contentClassAttribute) {
+            $io->comment(sprintf('Migrate %s:%s attribute.', $contentClassAttribute['contenttype_identifier'], $contentClassAttribute['identifier']));
+
+            try {
+                $xml = new SimpleXMLElement((string)$contentClassAttribute['columns']);
+
+                $isValidXml = true;
+            }
+            catch (Exception $e) {
+                $isValidXml = false;
+            }
+
+            if ($isValidXml) {
+                $columnList = $xml->xpath('//column-name');
+
+                $columns = [];
+
+                foreach ($columnList as $column) {
+                    $columns[(int)$column['idx']] = [
+                        'identifier' => (string)$column['id'],
+                        'name' => (string)$column,
+                    ];
+                }
+
+                $fieldDefinition = new FieldDefinition();
+                $storageFieldDefinition = new StorageFieldDefinition();
+
+                $fieldDefinition->fieldTypeConstraints->fieldSettings = [
+                    'minimum_rows' => $contentClassAttribute['minimum_rows'],
+                    'columns' => array_values($columns),
+                ];
+
+                $converter->toStorageFieldDefinition($fieldDefinition, $storageFieldDefinition);
+
+                $this->updateContentClassAttribute(
+                    (int)$contentClassAttribute['id'],
+                    (int)$storageFieldDefinition->dataInt1,
+                    (string)$storageFieldDefinition->dataText5
+                );
+
+
+                $columnsJson = $storageFieldDefinition->dataText5;
+            } else {
+                $columnsJson = $contentClassAttribute['columns'];
+            }
+
+            $contentAttributesCount = $this->getContentObjectAttributesCount(
+                (int)$contentClassAttribute['id']
+            );
+
+            $columns = json_decode($columnsJson);
+
+            $progressBar = $this->getProgressBar($contentAttributesCount, $output);
+            $progressBar->start();
+
+            for ($offset = 0; $offset <= $contentAttributesCount; $offset += $iterationCount) {
+                gc_disable();
+
+                $contentObjectAttributes = $this->getContentObjectAttributes(
+                    (int)$contentClassAttribute['id'],
+                    $offset,
+                    $iterationCount
+                );
+
+                foreach ($contentObjectAttributes as $contentObjectAttribute) {
+                    try {
+                        $xml = new SimpleXMLElement(
+                            (string)$contentObjectAttribute['data_text']
+                        );
+                    }
+                    catch (Exception $e) {
+                        $progressBar->advance();
+
+                        continue;
+                    }
+
+                    $storageFieldValue = new StorageFieldValue();
+                    $fieldValue = new FieldValue([
+                        'data' => [
+                            'entries' => [],
+                        ]
+                    ]);
+
+                    $rows = $this->convertCellsToRows($xml->xpath('c'), $columns);
+
+                    $fieldValue->data['entries'] = $rows;
+
+                    $converter->toStorageValue($fieldValue, $storageFieldValue);
+
+                    $this->updateContentObjectAttribute(
+                        (int)$contentObjectAttribute['id'],
+                        (string)$storageFieldValue->dataText
+                    );
+
+                    $progressBar->advance();
+                }
+
+                gc_enable();
+            }
+
+            $progressBar->finish();
+
+            $output->writeln(['', '']);
+        }
+
+        $io->success('Done.');
+    }
+
+    /**
+     * @param array $cells
+     * @param array $columns
+     *
+     * @return array
+     */
+    private function convertCellsToRows(array $cells, array $columns): array
+    {
+        $row = [];
+        $rows = [];
+        $columnsCount = count($columns);
+
+        foreach ($cells as $index => $cell) {
+            $columnIndex = $index % $columnsCount;
+            $columnIdentifier = $columns[$columnIndex]->identifier;
+
+            $row[$columnIdentifier] = (string)$cell;
+
+            if ($columnIndex === $columnsCount - 1) {
+                $rows[] = $row;
+                $row = [];
+            }
+        }
+
+        return $rows;
+    }
+
+    /**
+     * @return array
+     */
+    private function getContentClassAttributes(): array
+    {
+        $query = $this->connection->createQueryBuilder();
+        $query
+            ->select([
+                'attr.id',
+                'attr.identifier',
+                'attr.data_int1 as minimum_rows',
+                'attr.data_text5 as columns',
+                'class.identifier as contenttype_identifier',
+            ])
+            ->from('ezcontentclass_attribute', 'attr')
+            ->join('attr', 'ezcontentclass', 'class', 'class.id = attr.contentclass_id')
+            ->where('attr.data_type_string = :identifier')
+            ->setParameter(':identifier', self::EZMATRIX_IDENTIFIER);
+
+        return $query->execute()->fetchAll();
+    }
+
+    /**
+     * @param int $id
+     * @param int $minimumRows
+     * @param string $columns
+     */
+    private function updateContentClassAttribute(int $id, int $minimumRows, string $columns): void
+    {
+        $query = $this->connection->createQueryBuilder();
+        $query
+            ->update('ezcontentclass_attribute', 'attr')
+            ->set('attr.data_int1', ':minimum_rows')
+            ->set('attr.data_text5', ':columns')
+            ->where('attr.id = :id')
+            ->setParameter(':id', $id)
+            ->setParameter(':minimum_rows', $minimumRows)
+            ->setParameter(':columns', $columns);
+
+        $query->execute();
+    }
+
+    /**
+     * @param int $id
+     *
+     * @return int
+     */
+    private function getContentObjectAttributesCount(int $id): int
+    {
+        $query = $this->connection->createQueryBuilder();
+        $query
+            ->select('count(1)')
+            ->from('ezcontentobject_attribute', 'attr')
+            ->where('attr.contentclassattribute_id = :class_attr_id')
+            ->setParameter(':class_attr_id', $id);
+
+        return (int)$query->execute()->fetchColumn(0);
+    }
+
+    /**
+     * @param int $id
+     * @param int $offset
+     * @param int $iterationCount
+     *
+     * @return array
+     */
+    private function getContentObjectAttributes(int $id, int $offset, int $iterationCount): array
+    {
+        $query = $this->connection->createQueryBuilder();
+        $query
+            ->select(['id', 'data_text'])
+            ->from('ezcontentobject_attribute', 'attr')
+            ->where('attr.contentclassattribute_id = :class_attr_id')
+            ->setParameter(':class_attr_id', $id)
+            ->setFirstResult($offset)
+            ->setMaxResults($iterationCount);
+
+        return $query->execute()->fetchAll();
+    }
+
+    /**
+     * @param int $id
+     * @param string $rows
+     */
+    private function updateContentObjectAttribute(int $id, string $rows): void
+    {
+        $query = $this->connection->createQueryBuilder();
+        $query
+            ->update('ezcontentobject_attribute', 'attr')
+            ->set('attr.data_text', ':rows')
+            ->where('attr.id = :id')
+            ->setParameter(':id', $id)
+            ->setParameter(':rows', $rows);
+
+        $query->execute();
+    }
+
+    /**
+     * @param int $maxSteps
+     * @param \Symfony\Component\Console\Output\OutputInterface $output
+     *
+     * @return \Symfony\Component\Console\Helper\ProgressBar
+     */
+    protected function getProgressBar(int $maxSteps, OutputInterface $output): ProgressBar
+    {
+        $progressBar = new ProgressBar($output, $maxSteps);
+        $progressBar->setFormat(
+            ' %current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%'
+        );
+
+        return $progressBar;
+    }
+}

src/bundle/Resources/config/services.yml

@@ -1,2 +1,3 @@
 imports:
     - { resource: services/fieldtype.yml }
+    - { resource: services/migration.yml }

src/bundle/Resources/config/services/fieldtype.yml

@@ -17,8 +17,6 @@ services:
         tags:
             - { name: ezpublish.fieldType, alias: '%ezplatform.fieldtype.matrix.identifier%' }
 
-    EzSystems\EzPlatformMatrixFieldtype\FieldType\Converter\StorageValue\StorageValueConverterFactory: ~
-
     EzSystems\EzPlatformMatrixFieldtype\FieldType\Converter\MatrixConverter:
         tags:
             - { name: ezpublish.storageEngine.legacy.converter, alias: '%ezplatform.fieldtype.matrix.identifier%' }

src/bundle/Resources/config/services/migration.yml

@@ -0,0 +1,7 @@
+services:
+    _defaults:
+        autoconfigure: true
+        autowire: true
+        public: false
+
+    EzSystems\EzPlatformMatrixFieldtypeBundle\Command\MigrateLegacyMatrixCommand: ~

src/bundle/Resources/schema/legacy_ezcontentclass_attribute.xsd

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+  ~ @copyright Copyright (C) eZ Systems AS. All rights reserved.
+  ~ @license For full copyright and license information view LICENSE file distributed with this source code.
+  -->
+
+<!-- Legacy ezmatrix format:
+
+<?xml version="1.0" encoding="utf-8"?>
+<ezmatrix>
+    <column-name id="name" idx="0">Name</column-name>
+    <column-name id="time" idx="1">Time</column-name>
+</ezmatrix>
+
+-->
+
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
+    <xs:element name="ezmatrix">
+        <xs:complexType>
+            <xs:sequence>
+                <xs:element name="column-name" maxOccurs="unbounded" minOccurs="0">
+                    <xs:complexType>
+                        <xs:simpleContent>
+                            <xs:extension base="xs:string">
+                                <xs:attribute name="id" type="xs:string" use="required"/>
+                                <xs:attribute name="idx" type="xs:integer" use="required"/>
+                            </xs:extension>
+                        </xs:simpleContent>
+                    </xs:complexType>
+                </xs:element>
+            </xs:sequence>
+        </xs:complexType>
+    </xs:element>
+</xs:schema>