From 5c676c2a52577e0a25d6fc5ad73ab78edda1f747 Mon Sep 17 00:00:00 2001 From: Nicolas Joubert Date: Mon, 12 Oct 2026 11:09:48 +0200 Subject: [PATCH] feat(task) #258 Add XmlStreamReaderTask: iterable task reading an XML file element by element with \XMLReader (constant memory), outputting each element named node_name as a \DOMElement in its own \DOMDocument, ready for XpathEvaluatorTransformer. Add documentation and tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 1 + docs/index.md | 1 + docs/reference/tasks/xml_reader_task.md | 2 + .../reference/tasks/xml_stream_reader_task.md | 74 +++++ src/Task/File/Xml/XmlStreamReaderTask.php | 187 ++++++++++++ .../Task/File/Xml/XmlStreamReaderTaskTest.php | 276 ++++++++++++++++++ 6 files changed, 541 insertions(+) create mode 100644 docs/reference/tasks/xml_stream_reader_task.md create mode 100644 src/Task/File/Xml/XmlStreamReaderTask.php create mode 100644 tests/Task/File/Xml/XmlStreamReaderTaskTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 92d744af..357c081f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Latest * [#143](https://github.com/cleverage/process-bundle/issues/143) Improve PHPStan level from 6 to 7, and fix the level 8 errors that do not require a signature change (the remaining ones, due to nullable return types such as `AbstractConfigurableTask::getOptions(): ?array`, will be fixed in v6.0). Default to a `ContextualOptionResolver` in ProcessState and to a property accessor in ConditionTrait when none is set. * [#94](https://github.com/cleverage/process-bundle/issues/94) TransformerTrait: the `transformers` option (TransformerTask, MappingTransformer, ArrayMapTransformer, CachedTransformer, RulesTransformer, generic transformers) also accepts a list, whose items are a transformer code without options (`- trim`) or a single `code: options` map (`- callback: {...}`), to chain the same transformer without `#` suffix. The map syntax is still supported. Update documentation, add tests. * [#117](https://github.com/cleverage/process-bundle/issues/117) MappingTransformer: writing a target property that is a nested path (e.g. `address.city`) on an array destination, which silently created an `address.city` literal key, is deprecated and will throw an `\UnexpectedValueException` in v6.0. Use `[address.city]` for a literal key or `[address][city]` for a nested array; a simple property name (e.g. `name`) is still added as a key. Update documentation, add tests. +* [#258](https://github.com/cleverage/process-bundle/issues/258) Add XmlStreamReaderTask: iterable task reading an XML file element by element with `\XMLReader` (constant memory), outputting each element named `node_name` as a `\DOMElement` in its own `\DOMDocument`, ready for XpathEvaluatorTransformer. The file path is set in the `file_path` option or given as input. Add documentation and tests. ## Fixes * [#143](https://github.com/cleverage/process-bundle/issues/143) Fix InputIteratorTask: an `\IteratorAggregate` input whose `getIterator()` does not return an `\Iterator` (e.g. another `\IteratorAggregate`) is iterated instead of failing with a `TypeError`. Update documentation, add tests. diff --git a/docs/index.md b/docs/index.md index 5e96db36..a03a9b3c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -50,6 +50,7 @@ - [JsonStreamWriterTask](reference/tasks/json_stream_writer_task.md) - File/XML - [XmlReaderTask](reference/tasks/xml_reader_task.md) + - [XmlStreamReaderTask](reference/tasks/xml_stream_reader_task.md) - [XmlWriterTask](reference/tasks/xml_writer_task.md) - File/Yaml - [YamlReaderTask](reference/tasks/yaml_reader_task.md) diff --git a/docs/reference/tasks/xml_reader_task.md b/docs/reference/tasks/xml_reader_task.md index e2f43796..0537296e 100644 --- a/docs/reference/tasks/xml_reader_task.md +++ b/docs/reference/tasks/xml_reader_task.md @@ -47,3 +47,5 @@ Notes * An invalid XML content (not well-formed, undefined namespace prefix...) throws an `\UnexpectedValueException` whose message contains the libxml error(s) with their line and column. libxml warnings are tolerated. The libxml internal errors setting is restored after loading. +* The whole file is loaded in memory. Use [XmlStreamReaderTask](xml_stream_reader_task.md) to iterate over the elements + of a large file. diff --git a/docs/reference/tasks/xml_stream_reader_task.md b/docs/reference/tasks/xml_stream_reader_task.md new file mode 100644 index 00000000..f818ee28 --- /dev/null +++ b/docs/reference/tasks/xml_stream_reader_task.md @@ -0,0 +1,74 @@ +XmlStreamReaderTask +=================== + +Reads an XML file element by element with [XMLReader](https://www.php.net/manual/en/class.xmlreader.php) and iterates +over the elements with a given name, outputting each of them as a `\DOMElement`. Unlike +[XmlReaderTask](xml_reader_task.md), the whole file is never loaded in memory: use it for large XML files (product +catalogs, ERP exports...). +Requires the `dom` and `xmlreader` PHP extensions. + +Task reference +-------------- + +* **Service**: `CleverAge\ProcessBundle\Task\File\Xml\XmlStreamReaderTask` +* **Iterable task** + +Accepted inputs +--------------- + +`string`: path of the file to read, only used when the `file_path` option is not set (ignored otherwise). + +Possible outputs +---------------- + +`\DOMElement`: for each element named `node_name`, the element expanded with +[XMLReader::expand](https://www.php.net/manual/en/xmlreader.expand.php), as the root element of its own +`\DOMDocument`. It can be passed directly to the [XpathEvaluatorTransformer](../transformers/xpath_evaluator_transformer.md), with +queries relative to the element (e.g. `@id`, `name/text()`). + +When the file contains no matching element, no output is produced and the task is skipped. + +Options +------- + +| Code | Type | Required | Default | Description | +|-------------|----------------|:--------:|---------|--------------------------------------------------------------------------------------------------------------------------------------| +| `file_path` | `string\|null` | | `null` | Path of the file to read. When `null`, the path is read from the input | +| `node_name` | `string` | **X** | | Qualified name of the elements to iterate on, as written in the file (e.g. `product`, or `g:item` for an element with a prefix) | + +Examples +-------- + +```yaml +# Task configuration level +read: + service: '@CleverAge\ProcessBundle\Task\File\Xml\XmlStreamReaderTask' + options: + file_path: '%kernel.project_dir%/var/data/products.xml' + node_name: product + outputs: [transform] +transform: + service: '@CleverAge\ProcessBundle\Task\TransformerTask' + options: + transformers: + xpath_evaluator: + query: + sku: '@sku' + name: 'name/text()' + outputs: [write] +``` + +Notes +----- + +* Elements are matched at any depth. The subtree of a matching element is not searched again: an element named + `node_name` nested inside a matching element is part of its parent output, and is not output on its own. +* Namespaces declared on ancestors are copied to the expanded element, so prefixed queries (e.g. `g:price/text()`) + still work. +* An invalid XML content (not well-formed, empty file...) throws an `\UnexpectedValueException` whose message contains + the libxml error(s) with their line and column, handled by the task `error_strategy`. As the file is read as a + stream, the elements located before the error have already been output. libxml warnings are tolerated. +* A missing or unreadable file, or no file path in the option nor in the input, throws an `\UnexpectedValueException`. +* The file is released at the end of the iteration (and on `finalize()`), so that the task can read it again on its next + input. +* Use [XmlReaderTask](xml_reader_task.md) to get the whole document at once (small files). diff --git a/src/Task/File/Xml/XmlStreamReaderTask.php b/src/Task/File/Xml/XmlStreamReaderTask.php new file mode 100644 index 00000000..688bc79c --- /dev/null +++ b/src/Task/File/Xml/XmlStreamReaderTask.php @@ -0,0 +1,187 @@ +getOption($state, 'node_name'); + if ($this->reader instanceof \XMLReader) { + // Skip the subtree of the current element: nested elements with the same name are not iterated + $found = $this->moveToElement($nodeName, true); + } else { + $this->open($this->getFilePath($state)); + $found = $this->moveToElement($nodeName, false); + } + + if (!$found) { + // Release the file to allow the following iteration to read it again + $this->close(); + $state->setSkipped(true); + + return; + } + + $state->setOutput($this->expand()); + } + + /** + * The reader is moved in execute(), so that XML errors are handled by the error strategy of the task. + */ + public function next(ProcessState $state): bool + { + return $this->reader instanceof \XMLReader; + } + + public function finalize(ProcessState $state): void + { + $this->close(); + } + + protected function getFilePath(ProcessState $state): string + { + $filePath = $this->getOption($state, 'file_path') ?? $state->getInput(); + if (!\is_string($filePath) || '' === $filePath) { + throw new \UnexpectedValueException('No file path given in the "file_path" option or as input'); + } + + return $filePath; + } + + protected function open(string $filePath): void + { + if (!is_file($filePath) || !is_readable($filePath)) { + throw new \UnexpectedValueException(\sprintf('XML file "%s" does not exist or is not readable', $filePath)); + } + + $reader = new \XMLReader(); + if (!@$reader->open($filePath)) { + throw new \RuntimeException(\sprintf('Could not open XML file "%s"', $filePath)); + } + $this->reader = $reader; + $this->filePath = $filePath; + } + + /** + * Move the reader to the next element named $nodeName, return false at the end of the file. + */ + protected function moveToElement(string $nodeName, bool $skipCurrent): bool + { + $reader = $this->getReader(); + + $previousUseErrors = libxml_use_internal_errors(true); + libxml_clear_errors(); + try { + $found = $skipCurrent ? $reader->next() : $reader->read(); + while ($found && (\XMLReader::ELEMENT !== $reader->nodeType || $reader->name !== $nodeName)) { + $found = $reader->read(); + } + $errors = libxml_get_errors(); + } finally { + libxml_clear_errors(); + libxml_use_internal_errors($previousUseErrors); + } + + $this->throwOnErrors($errors); + + return $found; + } + + protected function expand(): \DOMElement + { + $reader = $this->getReader(); + $document = new \DOMDocument(); + + $previousUseErrors = libxml_use_internal_errors(true); + libxml_clear_errors(); + try { + $node = $reader->expand($document); + $errors = libxml_get_errors(); + } finally { + libxml_clear_errors(); + libxml_use_internal_errors($previousUseErrors); + } + + $this->throwOnErrors($errors); + if (!$node instanceof \DOMElement) { + throw new \UnexpectedValueException(\sprintf('Could not expand element "%s" of XML file "%s"', $reader->name, $this->filePath)); + } + + // Each element gets its own document, so that it can be evaluated with xpath like the output of XmlReaderTask + $document->appendChild($node); + + return $node; + } + + /** + * Warnings are tolerated, errors (e.g. mismatched tags) and fatal errors are not. + * + * @param array<\LibXMLError> $errors + */ + protected function throwOnErrors(array $errors): void + { + $errors = array_filter($errors, static fn (\LibXMLError $error): bool => \LIBXML_ERR_WARNING !== $error->level); + if ([] === $errors) { + return; + } + + $messages = array_map( + static fn (\LibXMLError $error): string => \sprintf('%s (line %d, column %d)', trim($error->message), $error->line, $error->column), + $errors, + ); + $filePath = $this->filePath; + $this->close(); + + throw new \UnexpectedValueException(\sprintf('Invalid XML in file "%s": %s', $filePath, implode('; ', $messages))); + } + + protected function getReader(): \XMLReader + { + if (!$this->reader instanceof \XMLReader) { + throw new \LogicException('No XML file opened'); + } + + return $this->reader; + } + + protected function close(): void + { + $this->reader?->close(); + $this->reader = null; + $this->filePath = null; + } + + protected function configureOptions(OptionsResolver $resolver): void + { + $resolver->setDefault('file_path', null); + $resolver->setAllowedTypes('file_path', ['string', 'null']); + + $resolver->setRequired('node_name'); + $resolver->setAllowedTypes('node_name', 'string'); + } +} diff --git a/tests/Task/File/Xml/XmlStreamReaderTaskTest.php b/tests/Task/File/Xml/XmlStreamReaderTaskTest.php new file mode 100644 index 00000000..0cbc6bb3 --- /dev/null +++ b/tests/Task/File/Xml/XmlStreamReaderTaskTest.php @@ -0,0 +1,276 @@ + + + + Foo + 10 + + + ignored + Bar20 + + XML; + + private string $tmpDir; + + protected function setUp(): void + { + $this->tmpDir = sys_get_temp_dir().\DIRECTORY_SEPARATOR.uniqid('xml_stream_reader_test_', true); + (new Filesystem())->dumpFile($this->tmpDir.'/catalog.xml', self::CATALOG); + } + + protected function tearDown(): void + { + (new Filesystem())->remove($this->tmpDir); + } + + public function testOutputsEachMatchingElement(): void + { + $outputs = $this->iterate(['file_path' => $this->tmpDir.'/catalog.xml', 'node_name' => 'product']); + + // The nested product is part of the first product and is not iterated + self::assertCount(2, $outputs); + self::assertSame(['1', '2'], array_map(static fn (\DOMElement $element): string => $element->getAttribute('id'), $outputs)); + self::assertSame('Bar', $outputs[1]->getElementsByTagName('name')->item(0)?->textContent); + } + + public function testEachElementIsTheRootOfItsOwnDocument(): void + { + $outputs = $this->iterate(['file_path' => $this->tmpDir.'/catalog.xml', 'node_name' => 'product']); + + self::assertNotSame($outputs[0]->ownerDocument, $outputs[1]->ownerDocument); + foreach ($outputs as $output) { + self::assertSame($output, $output->ownerDocument?->documentElement); + } + } + + public function testElementsCanBeEvaluatedWithXpathIncludingAncestorNamespaces(): void + { + $outputs = $this->iterate(['file_path' => $this->tmpDir.'/catalog.xml', 'node_name' => 'product']); + + $transformer = new XpathEvaluatorTransformer(); + $resolver = new OptionsResolver(); + $transformer->configureOptions($resolver); + $options = $resolver->resolve(['query' => ['id' => '@id', 'name' => 'name/text()', 'price' => 'g:price/text()']]); + + self::assertSame(['id' => '1', 'name' => 'Foo', 'price' => '10'], $transformer->transform($outputs[0], $options)); + self::assertSame(['id' => '2', 'name' => 'Bar', 'price' => '20'], $transformer->transform($outputs[1], $options)); + } + + public function testNodeNameIsTheQualifiedName(): void + { + $outputs = $this->iterate(['file_path' => $this->tmpDir.'/catalog.xml', 'node_name' => 'g:price']); + + self::assertSame(['10', '20'], array_map(static fn (\DOMElement $element): string => $element->textContent, $outputs)); + } + + public function testFilePathIsReadFromInputWhenNotSet(): void + { + $outputs = $this->iterate(['node_name' => 'product'], $this->tmpDir.'/catalog.xml'); + + self::assertCount(2, $outputs); + } + + public function testFilePathOptionHasPriorityOverInput(): void + { + $outputs = $this->iterate(['file_path' => $this->tmpDir.'/catalog.xml', 'node_name' => 'product'], $this->tmpDir.'/missing.xml'); + + self::assertCount(2, $outputs); + } + + public function testSkippedWhenNoElementMatches(): void + { + $task = new XmlStreamReaderTask(); + $state = $this->createState(['file_path' => $this->tmpDir.'/catalog.xml', 'node_name' => 'unknown']); + $task->initialize($state); + $task->execute($state); + + self::assertTrue($state->isSkipped()); + self::assertNull($state->getOutput()); + self::assertFalse($task->next($state)); + } + + public function testFileIsReadAgainOnTheFollowingExecution(): void + { + $task = new XmlStreamReaderTask(); + $state = $this->createState(['file_path' => $this->tmpDir.'/catalog.xml', 'node_name' => 'product']); + $task->initialize($state); + + self::assertCount(2, $this->iterateTask($task, $state)); + self::assertCount(2, $this->iterateTask($task, $state)); + } + + public function testFinalizeReleasesTheFile(): void + { + $task = new XmlStreamReaderTask(); + $state = $this->createState(['file_path' => $this->tmpDir.'/catalog.xml', 'node_name' => 'product']); + $task->initialize($state); + $task->execute($state); + self::assertTrue($task->next($state)); + + $task->finalize($state); + + self::assertFalse($task->next($state)); + $task->execute($state); + self::assertInstanceOf(\DOMElement::class, $state->getOutput()); + self::assertSame('1', $state->getOutput()->getAttribute('id')); + } + + public function testNextIsFalseBeforeTheFirstExecution(): void + { + self::assertFalse((new XmlStreamReaderTask())->next($this->createState(['node_name' => 'product']))); + } + + public function testInvalidXmlThrowsAndReleasesTheFile(): void + { + file_put_contents($this->tmpDir.'/invalid.xml', ''); + + $task = new XmlStreamReaderTask(); + $state = $this->createState(['file_path' => $this->tmpDir.'/invalid.xml', 'node_name' => 'product']); + $task->initialize($state); + + try { + $this->iterateTask($task, $state); + self::fail('An exception should have been thrown'); + } catch (\UnexpectedValueException $exception) { + self::assertStringContainsString('Invalid XML in file "'.$this->tmpDir.'/invalid.xml"', $exception->getMessage()); + self::assertStringContainsString('line 1', $exception->getMessage()); + } + self::assertFalse($task->next($state)); + } + + public function testEmptyFileThrows(): void + { + file_put_contents($this->tmpDir.'/empty.xml', ''); + + $this->expectException(\UnexpectedValueException::class); + $this->expectExceptionMessage('Invalid XML in file'); + + $this->iterate(['file_path' => $this->tmpDir.'/empty.xml', 'node_name' => 'product']); + } + + public function testMissingFileThrows(): void + { + $this->expectException(\UnexpectedValueException::class); + $this->expectExceptionMessage('does not exist or is not readable'); + + $this->iterate(['file_path' => $this->tmpDir.'/missing.xml', 'node_name' => 'product']); + } + + public function testMissingFilePathThrows(): void + { + $this->expectException(\UnexpectedValueException::class); + $this->expectExceptionMessage('No file path given'); + + $this->iterate(['node_name' => 'product']); + } + + public function testNodeNameIsRequired(): void + { + $this->expectException(MissingOptionsException::class); + + $this->iterate(['file_path' => $this->tmpDir.'/catalog.xml']); + } + + public function testNodeNameMustBeAString(): void + { + $this->expectException(InvalidOptionsException::class); + + $this->iterate(['file_path' => $this->tmpDir.'/catalog.xml', 'node_name' => 42]); + } + + public function testFilePathMustBeAString(): void + { + $this->expectException(InvalidOptionsException::class); + + $this->iterate(['file_path' => 42, 'node_name' => 'product']); + } + + /** + * @param array $options + * + * @return list<\DOMElement> + */ + private function iterate(array $options, mixed $input = null): array + { + $task = new XmlStreamReaderTask(); + $state = $this->createState($options); + $state->setInput($input); + $task->initialize($state); + + return $this->iterateTask($task, $state); + } + + /** + * @return list<\DOMElement> + */ + private function iterateTask(XmlStreamReaderTask $task, ProcessState $state): array + { + $outputs = []; + do { + $state->setSkipped(false); + $state->setOutput(null); + $task->execute($state); + if (!$state->isSkipped()) { + $output = $state->getOutput(); + self::assertInstanceOf(\DOMElement::class, $output); + $outputs[] = $output; + } + } while ($task->next($state)); + + return $outputs; + } + + /** + * @param array $options + */ + private function createState(array $options): ProcessState + { + $processConfiguration = new ProcessConfiguration('test', []); + $state = new ProcessState($processConfiguration, new ProcessHistory($processConfiguration)); + $state->setContextualOptionResolver(new ContextualOptionResolver()); + $state->setContext([]); + $state->setTaskConfiguration(new TaskConfiguration('read', XmlStreamReaderTask::class, $options)); + + return $state; + } +}