feat: serve message fetches from the cache

Signed-off-by: Sebastian Krupinski <krupinski01@gmail.com>

lib/Providers/CachedService.php

@@ -173,14 +173,33 @@ class CachedService extends ServiceBase
 
     public function entityFetchBulk(EntityIdentifierInterface ...$identifiers): array
     {
-        // served from the cache in 5.2c
-        return $this->live()->entityFetchBulk(...$identifiers);
+        return iterator_to_array($this->entityFetchStream(...$identifiers), true);
     }
 
+    /**
+     * Fetch messages from the cache; messages that are not cached are fetched from the
+     * server and cached, mailboxes never harmonized are read from the server.
+     */
     public function entityFetchStream(EntityIdentifierInterface ...$identifiers): Generator
     {
-        // served from the cache in 5.2c
-        return $this->live()->entityFetchStream(...$identifiers);
+        $byMailbox = [];
+        foreach ($identifiers as $identifier) {
+            if ($identifier->provider() !== $this->provider() || (string) $identifier->service() !== $this->serviceId()) {
+                throw new \InvalidArgumentException('Entity identifier does not belong to this service: ' . $identifier);
+            }
+            $byMailbox[(string) $identifier->collection()][] = (int) $identifier->entity();
+        }
+
+        foreach ($byMailbox as $mailbox => $uids) {
+            $uidValidity = $this->mailboxStore->state($this->serviceId(), (string) $mailbox)['uidValidity'];
+            $entities = $uidValidity === null
+                ? $this->liveMail()->entityFetch((string) $mailbox, ...$uids)
+                : $this->hydrate((string) $mailbox, (int) $uidValidity, $uids);
+
+            foreach ($entities as $entity) {
+                yield $entity->urn() => $entity;
+            }
+        }
     }
 
     /**
@@ -289,6 +308,9 @@ class CachedService extends ServiceBase
                     $missing[] = $uid;
                     continue;
                 }
+                if ($entity->getProperties()->getIncompleteSections() !== []) {
+                    $this->completeSections($mailbox, $uidValidity, $entity);
+                }
                 $entities[$uid] = $entity;
             }
 
@@ -307,6 +329,31 @@ class CachedService extends ServiceBase
         }
     }
 
+    /**
+     * Fetch the full content of sections cut off at ingest and store the completed message.
+     *
+     * Content is immutable, so the meta document and change sequence are untouched. A
+     * failure leaves the cut off text in place; it is tried again on the next read.
+     */
+    private function completeSections(string $mailbox, int $uidValidity, EntityResource $entity): void
+    {
+        $properties = $entity->getProperties();
+        $uid = (int) $entity->identifier();
+
+        try {
+            $sections = $this->liveMail()->messageSections($mailbox, $uid, ...$properties->getIncompleteSections());
+            if ($sections === []) {
+                return;
+            }
+            foreach ($sections as $partId => $content) {
+                $properties->completeSection((string) $partId, $content);
+            }
+            $this->fileStore->write($this->tenantId(), $this->serviceId(), $mailbox, $uidValidity, $uid, $entity->toCacheContent());
+        } catch (\Throwable) {
+            // keep the cut off text; retried on the next read
+        }
+    }
+
     /**
      * An entity from its meta document and message.json; null when the content is missing or outdated.
      */

lib/Providers/MessageProperties.php

@@ -139,6 +139,25 @@ class MessageProperties extends MessagePropertiesMutableAbstract {
         return $this;
     }
 
+    /**
+     * Replace the (cut off) content of a body section with its full content and clear its incomplete mark.
+     *
+     * @return bool whether the section exists in the body
+     */
+    public function completeSection(string $partId, string $content): bool
+    {
+        if (!isset($this->data[static::PROPERTY_BODY]) || !is_array($this->data[static::PROPERTY_BODY])) {
+            return false;
+        }
+
+        $found = self::replaceSectionContent($this->data[static::PROPERTY_BODY], $partId, $content);
+        if ($found) {
+            $this->incompleteSections = array_values(array_diff($this->incompleteSections, [$partId]));
+        }
+
+        return $found;
+    }
+
     /**
      * Serialise the fields the meta store needs for list, filter and sort.
      *
@@ -209,6 +228,23 @@ class MessageProperties extends MessagePropertiesMutableAbstract {
         return $this;
     }
 
+    private static function replaceSectionContent(array &$part, string $partId, string $content): bool
+    {
+        if ((string) ($part['partId'] ?? '') === $partId) {
+            $part['content'] = $content;
+            return true;
+        }
+
+        foreach ($part['subParts'] ?? [] as $index => $child) {
+            if (is_array($child) && self::replaceSectionContent($child, $partId, $content)) {
+                $part['subParts'][$index] = $child;
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     private function cachePreview(int $length = 200): string
     {
         $text = $this->getBodyTextPlain();

lib/Service/Live/LiveMailService.php

@@ -480,6 +480,31 @@ class LiveMailService
         }
     }
 
+    /**
+     * Fetch the full, decoded content of body sections of one message.
+     *
+     * @param string[] $partIds e.g. ['1.2']
+     * @return array<string, string> content keyed by part id; sections the server did not return are absent
+     */
+    public function messageSections(string $collection, int $uid, string ...$partIds): array
+    {
+        if ($partIds === []) {
+            return [];
+        }
+
+        // BODYSTRUCTURE lets the parser decode each section by its transfer encoding and charset
+        $options = FetchOptions::of('BODYSTRUCTURE');
+        foreach ($partIds as $partId) {
+            $options = $options->withBodySection($partId);
+        }
+
+        foreach ($this->messageFetch($collection, $options, $uid) as $message) {
+            return array_intersect_key($message->bodySections(), array_flip($partIds));
+        }
+
+        return [];
+    }
+
     /**
      * Stream the raw bytes of a message or a specific MIME part without buffering.
      *

tests/php/Support/FakeLiveMailService.php

@@ -96,6 +96,15 @@ final class FakeLiveMailService extends LiveMailService
         return $this->found;
     }
 
+    /** @var array<int, array<string, string>> full section content by UID */
+    public array $sections = [];
+
+    public function messageSections(string $collection, int $uid, string ...$partIds): array
+    {
+        $this->calls[] = ['messageSections', [$collection, $uid, $partIds]];
+        return array_intersect_key($this->sections[$uid] ?? [], array_flip($partIds));
+    }
+
     public function entityPatch(string $collection, array $flagsToAdd = [], array $flagsToRemove = [], int ...$uids): void
     {
         $this->calls[] = ['entityPatch', [$collection, $flagsToAdd, $flagsToRemove, $uids]];

tests/php/Support/FakeMessageFileStore.php

@@ -36,6 +36,12 @@ final class FakeMessageFileStore extends MessageFileStore
         $this->files = [];
     }
 
+    /** Direct access for tests */
+    public function content(int $uidValidity, int $uid): ?array
+    {
+        return $this->files[$uidValidity][$uid] ?? null;
+    }
+
     /** @return int[] */
     public function uids(int $uidValidity): array
     {

tests/php/Unit/CachedServiceTest.php

@@ -255,6 +255,81 @@ final class CachedServiceTest extends TestCase
         $this->assertCount(2, $this->service->entityListBulk('INBOX'));
     }
 
+    public function testFetchComesFromTheCache(): void
+    {
+        $this->harmonized([1 => [], 2 => ['\\Flagged']]);
+        $this->live->fetchBatches = [];
+
+        $entities = array_values($this->service->entityFetchBulk(
+            new EntityIdentifier('imap', 'svc', 'INBOX', '2'),
+        ));
+
+        $this->assertCount(1, $entities);
+        $this->assertSame(['flagged' => true], $entities[0]->getProperties()->getFlags());
+        $this->assertSame([], $this->live->fetchBatches);
+    }
+
+    public function testFetchOfAnUncachedMessageGoesToTheServerAndCaches(): void
+    {
+        $this->harmonized([1 => []]);
+        $this->live->flags[5] = [];
+        $this->live->fetchBatches = [];
+
+        $entities = array_values($this->service->entityFetchBulk(new EntityIdentifier('imap', 'svc', 'INBOX', '5')));
+
+        $this->assertSame('5', (string) $entities[0]->identifier());
+        $this->assertSame([[5]], $this->live->fetchBatches);
+        $this->assertContains(5, $this->files->uids(7));
+    }
+
+    public function testFetchFromANeverHarmonizedMailboxIsReadFromTheServer(): void
+    {
+        $this->live->flags = [3 => []];
+
+        $entities = array_values($this->service->entityFetchBulk(new EntityIdentifier('imap', 'svc', 'Sent', '3')));
+
+        $this->assertSame('3', (string) $entities[0]->identifier());
+        $this->assertSame([], $this->files->uids(7), 'nothing is cached without a mailbox generation');
+    }
+
+    public function testFetchRejectsIdentifiersOfAnotherService(): void
+    {
+        $this->expectException(\InvalidArgumentException::class);
+        iterator_to_array($this->service->entityFetchStream(new EntityIdentifier('imap', 'other', 'INBOX', '1')));
+    }
+
+    public function testIncompleteSectionsAreCompletedOnceAndStored(): void
+    {
+        $this->harmonized([1 => []]);
+        $content = $this->files->content(7, 1);
+        $content['incomplete'] = ['1'];
+        $content['properties']['body'] = ['partId' => '1', 'type' => 'text/plain', 'content' => 'Hello wor'];
+        $this->files->write('tenant', 'svc', 'INBOX', 7, 1, $content);
+        $this->live->sections = [1 => ['1' => 'Hello world, the full text.']];
+
+        $listed = array_values(iterator_to_array($this->service->entityListStream('INBOX'), false));
+        $fetched = array_values($this->service->entityFetchBulk(new EntityIdentifier('imap', 'svc', 'INBOX', '1')));
+
+        $this->assertSame('Hello world, the full text.', $listed[0]->getProperties()->getBodyTextPlain());
+        $this->assertSame('Hello world, the full text.', $fetched[0]->getProperties()->getBodyTextPlain());
+        $this->assertSame([], $this->files->content(7, 1)['incomplete']);
+        $this->assertCount(1, $this->calls('messageSections'), 'completed once, then served from the cache');
+    }
+
+    public function testIncompleteSectionsStayCutOffWhenTheServerDoesNotReturnThem(): void
+    {
+        $this->harmonized([1 => []]);
+        $content = $this->files->content(7, 1);
+        $content['incomplete'] = ['1'];
+        $content['properties']['body'] = ['partId' => '1', 'type' => 'text/plain', 'content' => 'Hello wor'];
+        $this->files->write('tenant', 'svc', 'INBOX', 7, 1, $content);
+
+        $fetched = array_values($this->service->entityFetchBulk(new EntityIdentifier('imap', 'svc', 'INBOX', '1')));
+
+        $this->assertSame('Hello wor', $fetched[0]->getProperties()->getBodyTextPlain());
+        $this->assertSame(['1'], $this->files->content(7, 1)['incomplete']);
+    }
+
     /**
      * Harmonize INBOX with the given server UIDs and flags.
      *

tests/php/Unit/MessageSectionsTest.php

@@ -0,0 +1,59 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KTXT\ProviderImap\Tests\Unit;
+
+use Generator;
+use KTXM\ProviderImap\Client\Protocol\Command\Argument\FetchOptions;
+use KTXM\ProviderImap\Client\Protocol\Parser\FetchMessageParser;
+use KTXM\ProviderImap\Providers\LiveService;
+use KTXM\ProviderImap\Providers\MessageProperties;
+use KTXM\ProviderImap\Service\Live\LiveMailService;
+use PHPUnit\Framework\TestCase;
+
+final class MessageSectionsTest extends TestCase
+{
+    public function testSectionsAreFetchedWithStructureAndDecoded(): void
+    {
+        $live = new class(new LiveService()) extends LiveMailService {
+            public ?string $command = null;
+
+            public function messageFetch(string $collection, ?FetchOptions $options = null, int ...$uids): Generator
+            {
+                $this->command = $options?->toCommand();
+                $structure = '(("TEXT" "PLAIN" ("CHARSET" "UTF-8") NIL NIL "BASE64" 16 1)'
+                    . '("TEXT" "HTML" ("CHARSET" "UTF-8") NIL NIL "7BIT" 12 1) "ALTERNATIVE" ("BOUNDARY" "b") NIL NIL)';
+                yield 7 => FetchMessageParser::parse(
+                    "* 1 FETCH (UID 7 BODYSTRUCTURE {$structure} BODY[1] {16}\r\nSGVsbG8gd29ybGQ= BODY[2] {12}\r\n<p>Hello</p>)"
+                );
+            }
+        };
+
+        $sections = $live->messageSections('INBOX', 7, '1', '2');
+
+        $this->assertStringContainsString('BODYSTRUCTURE', (string) $live->command);
+        $this->assertStringContainsString('BODY.PEEK[1]', (string) $live->command);
+        $this->assertStringContainsString('BODY.PEEK[2]', (string) $live->command);
+        $this->assertSame(['1' => 'Hello world', '2' => '<p>Hello</p>'], $sections);
+    }
+
+    public function testCompleteSectionReplacesNestedContentAndClearsTheMark(): void
+    {
+        $properties = new MessageProperties([]);
+        $properties->jsonDeserialize(['body' => [
+            'partId' => '1', 'type' => 'multipart/alternative', 'subParts' => [
+                ['partId' => '1.1', 'type' => 'text/plain', 'content' => 'Hi'],
+                ['partId' => '1.2', 'type' => 'text/html', 'content' => '<p>Hello wor'],
+            ],
+        ]]);
+        $properties->setIncompleteSections('1.2');
+
+        $this->assertTrue($properties->completeSection('1.2', '<p>Hello world</p>'));
+        $this->assertFalse($properties->completeSection('9', 'x'));
+
+        $this->assertSame('<p>Hello world</p>', $properties->getBodyTextHtml());
+        $this->assertSame('Hi', $properties->getBodyTextPlain());
+        $this->assertSame([], $properties->getIncompleteSections());
+    }
+}