Fix failover handling of chunked api commands (#1295)

fixes #1292

application/forms/Command/Object/AcknowledgeProblemForm.php

@@ -20,8 +20,6 @@ use ipl\Validator\CallbackValidator;
 use ipl\Web\FormDecorator\IcingaFormDecorator;
 use ipl\Web\Widget\Icon;
 use Iterator;
-use LimitIterator;
-use NoRewindIterator;
 use Traversable;
 
 use function ipl\Stdlib\iterable_value_first;
@@ -220,9 +218,9 @@ class AcknowledgeProblemForm extends CommandForm
         }
 
         $granted->rewind(); // Forwards the pointer to the first element
-        while ($granted->valid()) {
+        if ($granted->valid()) {
             // Chunk objects to avoid timeouts with large sets
-            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 250));
+            yield $command->setObjects($granted)->setChunkSize(250);
         }
     }
 }

application/forms/Command/Object/AddCommentForm.php

@@ -20,8 +20,6 @@ use ipl\Validator\CallbackValidator;
 use ipl\Web\FormDecorator\IcingaFormDecorator;
 use ipl\Web\Widget\Icon;
 use Iterator;
-use LimitIterator;
-use NoRewindIterator;
 use Traversable;
 
 use function ipl\Stdlib\iterable_value_first;
@@ -163,9 +161,9 @@ class AddCommentForm extends CommandForm
         }
 
         $granted->rewind(); // Forwards the pointer to the first element
-        while ($granted->valid()) {
+        if ($granted->valid()) {
             // Chunk objects to avoid timeouts with large sets
-            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 500));
+            yield $command->setObjects($granted)->setChunkSize(500);
         }
     }
 }

application/forms/Command/Object/CheckNowForm.php

@@ -11,8 +11,6 @@ use Icinga\Web\Notification;
 use ipl\Orm\Model;
 use ipl\Web\Widget\Icon;
 use Iterator;
-use LimitIterator;
-use NoRewindIterator;
 use Traversable;
 
 class CheckNowForm extends CommandForm
@@ -63,9 +61,9 @@ class CheckNowForm extends CommandForm
         $command->setForced();
 
         $granted->rewind(); // Forwards the pointer to the first element
-        while ($granted->valid()) {
+        if ($granted->valid()) {
             // Chunk objects to avoid timeouts with large sets
-            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 1000));
+            yield $command->setObjects($granted)->setChunkSize(1000);
         }
     }
 }

application/forms/Command/Object/DeleteCommentForm.php

@@ -11,8 +11,6 @@ use Icinga\Web\Notification;
 use ipl\Orm\Model;
 use ipl\Web\Widget\Icon;
 use Iterator;
-use LimitIterator;
-use NoRewindIterator;
 use Traversable;
 
 class DeleteCommentForm extends CommandForm
@@ -64,9 +62,9 @@ class DeleteCommentForm extends CommandForm
         $command->setAuthor($this->getAuth()->getUser()->getUsername());
 
         $granted->rewind(); // Forwards the pointer to the first element
-        while ($granted->valid()) {
+        if ($granted->valid()) {
             // Chunk objects to avoid timeouts with large sets
-            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 500));
+            yield $command->setObjects($granted)->setChunkSize(500);
         }
     }
 }

application/forms/Command/Object/DeleteDowntimeForm.php

@@ -11,8 +11,6 @@ use Icinga\Web\Notification;
 use ipl\Orm\Model;
 use ipl\Web\Widget\Icon;
 use Iterator;
-use LimitIterator;
-use NoRewindIterator;
 use Traversable;
 
 class DeleteDowntimeForm extends CommandForm
@@ -77,9 +75,9 @@ class DeleteDowntimeForm extends CommandForm
         $command->setAuthor($this->getAuth()->getUser()->getUsername());
 
         $granted->rewind(); // Forwards the pointer to the first element
-        while ($granted->valid()) {
+        if ($granted->valid()) {
             // Chunk objects to avoid timeouts with large sets
-            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 250));
+            yield $command->setObjects($granted)->setChunkSize(250);
         }
     }
 }

application/forms/Command/Object/ProcessCheckResultForm.php

@@ -16,8 +16,6 @@ use ipl\Orm\Model;
 use ipl\Web\FormDecorator\IcingaFormDecorator;
 use ipl\Web\Widget\Icon;
 use Iterator;
-use LimitIterator;
-use NoRewindIterator;
 use Traversable;
 
 use function ipl\Stdlib\iterable_value_first;
@@ -154,9 +152,9 @@ class ProcessCheckResultForm extends CommandForm
         $command->setPerformanceData($this->getValue('perfdata'));
 
         $granted->rewind(); // Forwards the pointer to the first element
-        while ($granted->valid()) {
+        if ($granted->valid()) {
             // Chunk objects to avoid timeouts with large sets
-            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 250));
+            yield $command->setObjects($granted)->setChunkSize(250);
         }
     }
 }

application/forms/Command/Object/RemoveAcknowledgementForm.php

@@ -12,8 +12,6 @@ use Icinga\Web\Notification;
 use ipl\Orm\Model;
 use ipl\Web\Widget\Icon;
 use Iterator;
-use LimitIterator;
-use NoRewindIterator;
 use Traversable;
 
 use function ipl\Stdlib\iterable_value_first;
@@ -77,9 +75,9 @@ class RemoveAcknowledgementForm extends CommandForm
         $command->setAuthor($this->getAuth()->getUser()->getUsername());
 
         $granted->rewind(); // Forwards the pointer to the first element
-        while ($granted->valid()) {
+        if ($granted->valid()) {
             // Chunk objects to avoid timeouts with large sets
-            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 250));
+            yield $command->setObjects($granted)->setChunkSize(250);
         }
     }
 }

application/forms/Command/Object/ScheduleCheckForm.php

@@ -18,8 +18,6 @@ use ipl\Orm\Model;
 use ipl\Web\FormDecorator\IcingaFormDecorator;
 use ipl\Web\Widget\Icon;
 use Iterator;
-use LimitIterator;
-use NoRewindIterator;
 use Traversable;
 
 use function ipl\Stdlib\iterable_value_first;
@@ -129,9 +127,9 @@ class ScheduleCheckForm extends CommandForm
         $command->setCheckTime($this->getValue('check_time')->getTimestamp());
 
         $granted->rewind(); // Forwards the pointer to the first element
-        while ($granted->valid()) {
+        if ($granted->valid()) {
             // Chunk objects to avoid timeouts with large sets
-            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 1000));
+            yield $command->setObjects($granted)->setChunkSize(1000);
         }
     }
 }

application/forms/Command/Object/ScheduleServiceDowntimeForm.php

@@ -19,8 +19,6 @@ use ipl\Validator\CallbackValidator;
 use ipl\Web\FormDecorator\IcingaFormDecorator;
 use ipl\Web\Widget\Icon;
 use Iterator;
-use LimitIterator;
-use NoRewindIterator;
 use Traversable;
 
 class ScheduleServiceDowntimeForm extends CommandForm
@@ -290,9 +288,9 @@ class ScheduleServiceDowntimeForm extends CommandForm
         }
 
         $granted->rewind(); // Forwards the pointer to the first element
-        while ($granted->valid()) {
+        if ($granted->valid()) {
             // Chunk objects to avoid timeouts with large sets
-            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 250));
+            yield $command->setObjects($granted)->setChunkSize(250);
         }
     }
 }

application/forms/Command/Object/SendCustomNotificationForm.php

@@ -17,8 +17,6 @@ use ipl\Orm\Model;
 use ipl\Web\FormDecorator\IcingaFormDecorator;
 use ipl\Web\Widget\Icon;
 use Iterator;
-use LimitIterator;
-use NoRewindIterator;
 use Traversable;
 
 use function ipl\Stdlib\iterable_value_first;
@@ -130,9 +128,9 @@ class SendCustomNotificationForm extends CommandForm
         $command->setAuthor($this->getAuth()->getUser()->getUsername());
 
         $granted->rewind(); // Forwards the pointer to the first element
-        while ($granted->valid()) {
+        if ($granted->valid()) {
             // Chunk objects to avoid timeouts with large sets
-            yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 500));
+            yield $command->setObjects($granted)->setChunkSize(500);
         }
     }
 }

application/forms/Command/Object/ToggleObjectFeaturesForm.php

@@ -12,8 +12,6 @@ use ipl\Html\FormElement\CheckboxElement;
 use ipl\Orm\Model;
 use ipl\Web\FormDecorator\IcingaFormDecorator;
 use Iterator;
-use LimitIterator;
-use NoRewindIterator;
 use Traversable;
 
 class ToggleObjectFeaturesForm extends CommandForm
@@ -181,11 +179,11 @@ class ToggleObjectFeaturesForm extends CommandForm
             $command->setEnabled((int) $state);
 
             $granted->rewind(); // Forwards the pointer to the first element
-            while ($granted->valid()) {
+            if ($granted->valid()) {
                 $this->submittedFeatures[$command->getFeature()] ??= $command->getEnabled();
 
                 // Chunk objects to avoid timeouts with large sets
-                yield $command->setObjects(new LimitIterator(new NoRewindIterator($granted), 0, 1000));
+                yield $command->setObjects($granted)->setChunkSize(1000);
             }
         }
     }

library/Icingadb/Command/Object/ObjectsCommand.php

@@ -9,6 +9,7 @@ use Generator;
 use Icinga\Module\Icingadb\Command\IcingaCommand;
 use InvalidArgumentException;
 use ipl\Orm\Model;
+use Iterator;
 use LogicException;
 use Traversable;
 
@@ -20,20 +21,27 @@ abstract class ObjectsCommand extends IcingaCommand
     /**
      * Involved objects
      *
-     * @var Traversable<Model>
+     * @var ?Iterator<Model>
      */
     protected $objects;
 
+    /**
+     * How many objects to process at once
+     *
+     * @var ?int
+     */
+    protected ?int $chunkSize = null;
+
     /**
      * Set the involved objects
      *
-     * @param Traversable<Model> $objects Except generators
+     * @param Iterator<Model> $objects Except generators
      *
      * @return $this
      *
      * @throws InvalidArgumentException If a generator is passed
      */
-    public function setObjects(Traversable $objects): self
+    public function setObjects(Iterator $objects): self
     {
         if ($objects instanceof Generator) {
             throw new InvalidArgumentException('Generators are not supported');
@@ -61,9 +69,9 @@ abstract class ObjectsCommand extends IcingaCommand
     /**
      * Get the involved objects
      *
-     * @return Traversable
+     * @return Iterator
      */
-    public function getObjects(): Traversable
+    public function getObjects(): Iterator
     {
         if ($this->objects === null) {
             throw new LogicException(
@@ -73,4 +81,28 @@ abstract class ObjectsCommand extends IcingaCommand
 
         return $this->objects;
     }
+
+    /**
+     * Set how many objects to process at once
+     *
+     * @param ?int $chunkSize
+     *
+     * @return $this
+     */
+    public function setChunkSize(?int $chunkSize): static
+    {
+        $this->chunkSize = $chunkSize;
+
+        return $this;
+    }
+
+    /**
+     * Get how many objects to process at once
+     *
+     * @return ?int
+     */
+    public function getChunkSize(): ?int
+    {
+        return $this->chunkSize;
+    }
 }

library/Icingadb/Command/Transport/ApiCommandTransport.php

@@ -256,11 +256,11 @@ class ApiCommandTransport implements CommandTransportInterface
                 ), static::SEND_TIMEOUT, $e);
             }
 
-            throw new CommandTransportException(
+            throw (new CommandTransportException(
                 'Can\'t connect to the Icinga 2 API: %u %s',
                 $e->getCode(),
                 $e->getMessage()
-            );
+            ))->setCommand($command);
         }
 
         try {
@@ -299,16 +299,20 @@ class ApiCommandTransport implements CommandTransportInterface
     /**
      * Send the Icinga command over the Icinga 2 API
      *
-     * @param   IcingaCommand   $command
-     * @param   int|null        $now
+     * @param IcingaCommand|IcingaApiCommand $command
+     * @param int|null $now
      *
-     * @throws  CommandTransportException
+     * @throws CommandTransportException
      *
-     * @return  mixed
+     * @return mixed
      */
-    public function send(IcingaCommand $command, int $now = null)
+    public function send(IcingaCommand|IcingaApiCommand $command, int $now = null)
     {
-        return $this->sendCommand($this->renderer->render($command));
+        if ($command instanceof IcingaCommand) {
+            $command = $this->renderer->render($command);
+        }
+
+        return $this->sendCommand($command);
     }
 
     /**

library/Icingadb/Command/Transport/CommandTransport.php

@@ -4,12 +4,12 @@
 
 namespace Icinga\Module\Icingadb\Command\Transport;
 
-use Exception;
 use Icinga\Application\Config;
 use Icinga\Application\Logger;
 use Icinga\Data\ConfigObject;
 use Icinga\Exception\ConfigurationError;
 use Icinga\Module\Icingadb\Command\IcingaCommand;
+use Icinga\Module\Icingadb\Command\Object\ObjectsCommand;
 
 /**
  * Command transport
@@ -103,19 +103,82 @@ class CommandTransport implements CommandTransportInterface
     public function send(IcingaCommand $command, int $now = null)
     {
         $errors = [];
+        $results = [];
+        $retryCommand = null;
 
         foreach (static::getConfig() as $name => $transportConfig) {
             $transport = static::createTransport($transportConfig);
 
-            try {
-                $result = $transport->send($command, $now);
-            } catch (CommandTransportException $e) {
-                Logger::error($e);
-                $errors[] = sprintf('%s: %s.', $name, rtrim($e->getMessage(), '.'));
-                continue; // Try the next transport
-            }
+            if ($command instanceof ObjectsCommand && $command->getChunkSize() > 0) {
+                $objects = $command->getObjects();
 
-            return $result; // The command was successfully sent
+                if ($retryCommand !== null) {
+                    try {
+                        $results[] = $transport->send($retryCommand, $now);
+                    } catch (CommandTransportException) {
+                        // It failed prior, so no need to log it again
+                        continue;
+                    }
+
+                    $retryCommand = null;
+                } else {
+                    if ($objects->key() === null) {
+                        // We traverse the iterator manually here, so we have to rewind it before the first iteration.
+                        // That should be the case if the current key is null. May fail if an iterator explicitly yields
+                        // null as the key, but I want to see a justified use case for that…
+                        $objects->rewind();
+                    }
+                }
+
+                while ($objects->valid()) {
+                    $batchCommand = clone $command;
+                    $batchCommand->setObjects(
+                        new \LimitIterator(new \NoRewindIterator($objects), 0, $command->getChunkSize())
+                    );
+
+                    try {
+                        $results[] = $transport->send($batchCommand, $now);
+                    } catch (CommandTransportException $e) {
+                        Logger::error($e);
+                        $errors[] = sprintf('%s: %s.', $name, rtrim($e->getMessage(), '.'));
+
+                        $retryCommand = $e->getCommand();
+                        if ($retryCommand !== null) {
+                            continue 2;
+                        } else {
+                            // Non-recoverable error, so stop trying to send further commands
+                            break 2;
+                        }
+                    }
+                }
+
+                return $results;
+            } elseif ($retryCommand !== null) {
+                try {
+                    $result = $transport->send($retryCommand, $now);
+                } catch (CommandTransportException) {
+                    // It failed prior, so no need to log it again
+                    continue;
+                }
+
+                return $result;
+            } else {
+                try {
+                    $result = $transport->send($command, $now);
+                } catch (CommandTransportException $e) {
+                    Logger::error($e);
+                    $errors[] = sprintf('%s: %s.', $name, rtrim($e->getMessage(), '.'));
+
+                    $retryCommand = $e->getCommand();
+                    if ($retryCommand !== null) {
+                        continue; // Try the next transport
+                    } else {
+                        break;
+                    }
+                }
+
+                return $result; // The command was successfully sent
+            }
         }
 
         if (! empty($errors)) {

library/Icingadb/Command/Transport/CommandTransportException.php

@@ -11,4 +11,33 @@ use Icinga\Exception\IcingaException;
  */
 class CommandTransportException extends IcingaException
 {
+    /** @var mixed The command that was not sent */
+    private mixed $command = null;
+
+    /**
+     * Set the command that was not sent
+     *
+     * This will be passed to the next transport in the chain.
+     * Make sure the transport accepts this type in {@see CommandTransportInterface::send()}.
+     *
+     * @param mixed $command
+     *
+     * @return $this
+     */
+    public function setCommand(mixed $command): static
+    {
+        $this->command = $command;
+
+        return $this;
+    }
+
+    /**
+     * Get the command that was not sent
+     *
+     * @return mixed
+     */
+    public function getCommand(): mixed
+    {
+        return $this->command;
+    }
 }

test/php/Lib/FailoverCommandTransport.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace Tests\Icinga\Module\Icingadb\Lib;
+
+use Icinga\Application\Config;
+use Icinga\Data\ConfigObject;
+use Icinga\Module\Icingadb\Command\IcingaApiCommand;
+use Icinga\Module\Icingadb\Command\Transport\ApiCommandTransport;
+use Icinga\Module\Icingadb\Command\Transport\CommandTransport;
+use Icinga\Module\Icingadb\Command\Transport\CommandTransportException;
+
+class FailoverCommandTransport extends CommandTransport
+{
+    public static function getConfig(): Config
+    {
+        return Config::fromArray(['endpoint1' => ['host' => 'endpointA'], 'endpoint2' => ['host' => 'endpointB']]);
+    }
+
+    public static function createTransport(ConfigObject $config): ApiCommandTransport
+    {
+        return (new class extends ApiCommandTransport {
+            protected function sendCommand(IcingaApiCommand $command)
+            {
+                if ($this->getHost() === 'endpointA') {
+                    throw (new CommandTransportException(sprintf('%s fails!', $this->getHost())))
+                        ->setCommand($command);
+                }
+
+                return $command->getData();
+            }
+        })->setHost($config->host);
+    }
+}

test/php/Lib/IntermittentlyFailingCommandTransport.php

@@ -0,0 +1,40 @@
+<?php
+
+namespace Tests\Icinga\Module\Icingadb\Lib;
+
+use Icinga\Application\Config;
+use Icinga\Data\ConfigObject;
+use Icinga\Module\Icingadb\Command\IcingaApiCommand;
+use Icinga\Module\Icingadb\Command\Transport\ApiCommandTransport;
+use Icinga\Module\Icingadb\Command\Transport\CommandTransport;
+use Icinga\Module\Icingadb\Command\Transport\CommandTransportException;
+
+class IntermittentlyFailingCommandTransport extends CommandTransport
+{
+    public static $failAtAttemptNo = 2;
+
+    public static $attemptNo = 0;
+
+    public static function getConfig(): Config
+    {
+        return Config::fromArray(['endpoint1' => ['host' => 'endpointA'], 'endpoint2' => ['host' => 'endpointB']]);
+    }
+
+    public static function createTransport(ConfigObject $config): ApiCommandTransport
+    {
+        return (new class extends ApiCommandTransport {
+            protected function sendCommand(IcingaApiCommand $command)
+            {
+                $attemptNo = ++IntermittentlyFailingCommandTransport::$attemptNo;
+                $failAtAttemptNo = IntermittentlyFailingCommandTransport::$failAtAttemptNo;
+
+                if ($attemptNo === $failAtAttemptNo) {
+                    throw (new CommandTransportException(sprintf('%s intermittently fails!', $this->getHost())))
+                        ->setCommand($command);
+                }
+
+                return $command->getData() + ['endpoint' => $this->getHost()];
+            }
+        })->setHost($config->host);
+    }
+}

test/php/library/Icingadb/Command/Transport/CommandTransportTest.php

@@ -6,14 +6,16 @@ use Icinga\Module\Icingadb\Command\Object\AddCommentCommand;
 use Icinga\Module\Icingadb\Command\Transport\CommandTransportException;
 use Icinga\Module\Icingadb\Model\Host;
 use PHPUnit\Framework\TestCase;
+use Tests\Icinga\Module\Icingadb\Lib\FailoverCommandTransport;
+use Tests\Icinga\Module\Icingadb\Lib\IntermittentlyFailingCommandTransport;
 use Tests\Icinga\Module\Icingadb\Lib\StrikingCommandTransport;
 
 class CommandTransportTest extends TestCase
 {
-    public function testFallbackHandling()
+    public function testFatalErrorHandling(): void
     {
         $this->expectException(CommandTransportException::class);
-        $this->expectExceptionMessage('endpointB strikes!');
+        $this->expectExceptionMessage('endpointA strikes!');
 
         (new StrikingCommandTransport())->send(
             (new AddCommentCommand())
@@ -29,7 +31,31 @@ class CommandTransportTest extends TestCase
         );
     }
 
-    public function testGeneratorsAreNotSupported()
+    public function testFallbackHandling(): void
+    {
+        $result = (new FailoverCommandTransport())->send(
+            (new AddCommentCommand())
+                ->setExpireTime(42)
+                ->setAuthor('GLaDOS')
+                ->setComment('The cake is a lie')
+                ->setObjects(new \ArrayIterator([
+                    (new Host())->setProperties(['name' => 'host1']),
+                    (new Host())->setProperties(['name' => 'host2']),
+                ]))
+        );
+
+        $this->assertSame(
+            [
+                'author' => 'GLaDOS',
+                'comment' => 'The cake is a lie',
+                'expiry' => 42,
+                'hosts' => ['host1', 'host2']
+            ],
+            $result
+        );
+    }
+
+    public function testGeneratorsAreNotSupported(): void
     {
         $this->expectException(\InvalidArgumentException::class);
         $this->expectExceptionMessage('Generators are not supported');
@@ -45,4 +71,200 @@ class CommandTransportTest extends TestCase
                 })())
         );
     }
+
+    public function testChunkedObjectsWithFallbackHandling(): void
+    {
+        // Multiple chunks
+
+        $result = (new FailoverCommandTransport())->send(
+            (new AddCommentCommand())
+                ->setExpireTime(42)
+                ->setAuthor('GLaDOS')
+                ->setComment('The cake is a lie')
+                ->setChunkSize(1)
+                ->setObjects(new \ArrayIterator([
+                    (new Host())->setProperties(['name' => 'host1']),
+                    (new Host())->setProperties(['name' => 'host2']),
+                ]))
+        );
+
+        $this->assertSame(
+            [
+                [
+                    'author' => 'GLaDOS',
+                    'comment' => 'The cake is a lie',
+                    'expiry' => 42,
+                    'hosts' => ['host1']
+                ],
+                [
+                    'author' => 'GLaDOS',
+                    'comment' => 'The cake is a lie',
+                    'expiry' => 42,
+                    'hosts' => ['host2']
+                ]
+            ],
+            $result
+        );
+
+        // A single chunk
+
+        $result = (new FailoverCommandTransport())->send(
+            (new AddCommentCommand())
+                ->setExpireTime(42)
+                ->setAuthor('GLaDOS')
+                ->setComment('The cake is a lie')
+                ->setChunkSize(4)
+                ->setObjects(new \ArrayIterator([
+                    (new Host())->setProperties(['name' => 'host1']),
+                    (new Host())->setProperties(['name' => 'host2']),
+                ]))
+        );
+
+        $this->assertSame(
+            [
+                [
+                    'author' => 'GLaDOS',
+                    'comment' => 'The cake is a lie',
+                    'expiry' => 42,
+                    'hosts' => ['host1', 'host2']
+                ]
+            ],
+            $result
+        );
+    }
+
+    public function testIntermittentFailureHandlingDuringChunkedTransmission(): void
+    {
+        // Fails after the 2nd chunk
+
+        $result = (new IntermittentlyFailingCommandTransport())->send(
+            (new AddCommentCommand())
+                ->setExpireTime(42)
+                ->setAuthor('GLaDOS')
+                ->setComment('The cake is a lie')
+                ->setChunkSize(1)
+                ->setObjects(new \CallbackFilterIterator(new \ArrayIterator([
+                    (new Host())->setProperties(['name' => 'host1']),
+                    (new Host())->setProperties(['name' => 'host2']),
+                    (new Host())->setProperties(['name' => 'host3']),
+                    (new Host())->setProperties(['name' => 'host4']),
+                    (new Host())->setProperties(['name' => 'host5']),
+                    (new Host())->setProperties(['name' => 'host6']),
+                ]), fn() => true))
+        );
+
+        $this->assertSame(
+            [
+                [
+                    'author' => 'GLaDOS',
+                    'comment' => 'The cake is a lie',
+                    'expiry' => 42,
+                    'hosts' => ['host1'],
+                    'endpoint' => 'endpointA'
+                ],
+                [
+                    'author' => 'GLaDOS',
+                    'comment' => 'The cake is a lie',
+                    'expiry' => 42,
+                    'hosts' => ['host2'],
+                    'endpoint' => 'endpointB'
+                ],
+                [
+                    'author' => 'GLaDOS',
+                    'comment' => 'The cake is a lie',
+                    'expiry' => 42,
+                    'hosts' => ['host3'],
+                    'endpoint' => 'endpointB'
+                ],
+                [
+                    'author' => 'GLaDOS',
+                    'comment' => 'The cake is a lie',
+                    'expiry' => 42,
+                    'hosts' => ['host4'],
+                    'endpoint' => 'endpointB'
+                ],
+                [
+                    'author' => 'GLaDOS',
+                    'comment' => 'The cake is a lie',
+                    'expiry' => 42,
+                    'hosts' => ['host5'],
+                    'endpoint' => 'endpointB'
+                ],
+                [
+                    'author' => 'GLaDOS',
+                    'comment' => 'The cake is a lie',
+                    'expiry' => 42,
+                    'hosts' => ['host6'],
+                    'endpoint' => 'endpointB'
+                ]
+            ],
+            $result
+        );
+
+        // Fails after the next-to-last chunk
+
+        IntermittentlyFailingCommandTransport::$failAtAttemptNo = 3;
+        IntermittentlyFailingCommandTransport::$attemptNo = 0;
+
+        $result = (new IntermittentlyFailingCommandTransport())->send(
+            (new AddCommentCommand())
+                ->setExpireTime(42)
+                ->setAuthor('GLaDOS')
+                ->setComment('The cake is a lie')
+                ->setChunkSize(2)
+                ->setObjects(new \CallbackFilterIterator(new \ArrayIterator([
+                    (new Host())->setProperties(['name' => 'host1']),
+                    (new Host())->setProperties(['name' => 'host2']),
+                    (new Host())->setProperties(['name' => 'host3']),
+                    (new Host())->setProperties(['name' => 'host4']),
+                    (new Host())->setProperties(['name' => 'host5']),
+                    (new Host())->setProperties(['name' => 'host6']),
+                ]), fn() => true))
+        );
+
+        $this->assertSame(
+            [
+                [
+                    'author' => 'GLaDOS',
+                    'comment' => 'The cake is a lie',
+                    'expiry' => 42,
+                    'hosts' => ['host1', 'host2'],
+                    'endpoint' => 'endpointA'
+                ],
+                [
+                    'author' => 'GLaDOS',
+                    'comment' => 'The cake is a lie',
+                    'expiry' => 42,
+                    'hosts' => ['host3', 'host4'],
+                    'endpoint' => 'endpointA'
+                ],
+                [
+                    'author' => 'GLaDOS',
+                    'comment' => 'The cake is a lie',
+                    'expiry' => 42,
+                    'hosts' => ['host5', 'host6'],
+                    'endpoint' => 'endpointB'
+                ]
+            ],
+            $result
+        );
+    }
+
+    public function testFatalErrorHandlingDuringChunkedTransmission(): void
+    {
+        $this->expectException(CommandTransportException::class);
+        $this->expectExceptionMessage('endpointA strikes!');
+
+        (new StrikingCommandTransport())->send(
+            (new AddCommentCommand())
+            ->setExpireTime(42)
+            ->setAuthor('GLaDOS')
+            ->setComment('The cake is a lie')
+            ->setChunkSize(1)
+            ->setObjects(new \ArrayIterator([
+                (new Host())->setProperties(['name' => 'host1']),
+                (new Host())->setProperties(['name' => 'host2']),
+            ]))
+        );
+    }
 }