From 5e458c5bc8646b44c415cda3ea604047cdb1e103 Mon Sep 17 00:00:00 2001 From: Sebastian Krupinski Date: Wed, 7 Oct 2026 22:20:13 -0400 Subject: [PATCH] feat: serve message fetches from the cache Signed-off-by: Sebastian Krupinski --- lib/Providers/CachedService.php | 55 ++++++++++++++-- lib/Providers/MessageProperties.php | 36 +++++++++++ lib/Service/Live/LiveMailService.php | 25 ++++++++ tests/php/Support/FakeLiveMailService.php | 9 +++ tests/php/Support/FakeMessageFileStore.php | 6 ++ tests/php/Unit/CachedServiceTest.php | 75 ++++++++++++++++++++++ tests/php/Unit/MessageSectionsTest.php | 59 +++++++++++++++++ 7 files changed, 261 insertions(+), 4 deletions(-) create mode 100644 tests/php/Unit/MessageSectionsTest.php diff --git a/lib/Providers/CachedService.php b/lib/Providers/CachedService.php index 90f0ebf..4d745a7 100644 --- a/lib/Providers/CachedService.php +++ b/lib/Providers/CachedService.php @@ -173,14 +173,33 @@ class CachedService extends ServiceBase public function entityFetchBulk(EntityIdentifierInterface ...$identifiers): array { - // served from the cache in 5.2c - return $this->live()->entityFetchBulk(...$identifiers); + return iterator_to_array($this->entityFetchStream(...$identifiers), true); } + /** + * Fetch messages from the cache; messages that are not cached are fetched from the + * server and cached, mailboxes never harmonized are read from the server. + */ public function entityFetchStream(EntityIdentifierInterface ...$identifiers): Generator { - // served from the cache in 5.2c - return $this->live()->entityFetchStream(...$identifiers); + $byMailbox = []; + foreach ($identifiers as $identifier) { + if ($identifier->provider() !== $this->provider() || (string) $identifier->service() !== $this->serviceId()) { + throw new \InvalidArgumentException('Entity identifier does not belong to this service: ' . $identifier); + } + $byMailbox[(string) $identifier->collection()][] = (int) $identifier->entity(); + } + + foreach ($byMailbox as $mailbox => $uids) { + $uidValidity = $this->mailboxStore->state($this->serviceId(), (string) $mailbox)['uidValidity']; + $entities = $uidValidity === null + ? $this->liveMail()->entityFetch((string) $mailbox, ...$uids) + : $this->hydrate((string) $mailbox, (int) $uidValidity, $uids); + + foreach ($entities as $entity) { + yield $entity->urn() => $entity; + } + } } /** @@ -289,6 +308,9 @@ class CachedService extends ServiceBase $missing[] = $uid; continue; } + if ($entity->getProperties()->getIncompleteSections() !== []) { + $this->completeSections($mailbox, $uidValidity, $entity); + } $entities[$uid] = $entity; } @@ -307,6 +329,31 @@ class CachedService extends ServiceBase } } + /** + * Fetch the full content of sections cut off at ingest and store the completed message. + * + * Content is immutable, so the meta document and change sequence are untouched. A + * failure leaves the cut off text in place; it is tried again on the next read. + */ + private function completeSections(string $mailbox, int $uidValidity, EntityResource $entity): void + { + $properties = $entity->getProperties(); + $uid = (int) $entity->identifier(); + + try { + $sections = $this->liveMail()->messageSections($mailbox, $uid, ...$properties->getIncompleteSections()); + if ($sections === []) { + return; + } + foreach ($sections as $partId => $content) { + $properties->completeSection((string) $partId, $content); + } + $this->fileStore->write($this->tenantId(), $this->serviceId(), $mailbox, $uidValidity, $uid, $entity->toCacheContent()); + } catch (\Throwable) { + // keep the cut off text; retried on the next read + } + } + /** * An entity from its meta document and message.json; null when the content is missing or outdated. */ diff --git a/lib/Providers/MessageProperties.php b/lib/Providers/MessageProperties.php index f9bbb38..a451f81 100644 --- a/lib/Providers/MessageProperties.php +++ b/lib/Providers/MessageProperties.php @@ -139,6 +139,25 @@ class MessageProperties extends MessagePropertiesMutableAbstract { return $this; } + /** + * Replace the (cut off) content of a body section with its full content and clear its incomplete mark. + * + * @return bool whether the section exists in the body + */ + public function completeSection(string $partId, string $content): bool + { + if (!isset($this->data[static::PROPERTY_BODY]) || !is_array($this->data[static::PROPERTY_BODY])) { + return false; + } + + $found = self::replaceSectionContent($this->data[static::PROPERTY_BODY], $partId, $content); + if ($found) { + $this->incompleteSections = array_values(array_diff($this->incompleteSections, [$partId])); + } + + return $found; + } + /** * Serialise the fields the meta store needs for list, filter and sort. * @@ -209,6 +228,23 @@ class MessageProperties extends MessagePropertiesMutableAbstract { return $this; } + private static function replaceSectionContent(array &$part, string $partId, string $content): bool + { + if ((string) ($part['partId'] ?? '') === $partId) { + $part['content'] = $content; + return true; + } + + foreach ($part['subParts'] ?? [] as $index => $child) { + if (is_array($child) && self::replaceSectionContent($child, $partId, $content)) { + $part['subParts'][$index] = $child; + return true; + } + } + + return false; + } + private function cachePreview(int $length = 200): string { $text = $this->getBodyTextPlain(); diff --git a/lib/Service/Live/LiveMailService.php b/lib/Service/Live/LiveMailService.php index 9736767..b9cd222 100644 --- a/lib/Service/Live/LiveMailService.php +++ b/lib/Service/Live/LiveMailService.php @@ -480,6 +480,31 @@ class LiveMailService } } + /** + * Fetch the full, decoded content of body sections of one message. + * + * @param string[] $partIds e.g. ['1.2'] + * @return array content keyed by part id; sections the server did not return are absent + */ + public function messageSections(string $collection, int $uid, string ...$partIds): array + { + if ($partIds === []) { + return []; + } + + // BODYSTRUCTURE lets the parser decode each section by its transfer encoding and charset + $options = FetchOptions::of('BODYSTRUCTURE'); + foreach ($partIds as $partId) { + $options = $options->withBodySection($partId); + } + + foreach ($this->messageFetch($collection, $options, $uid) as $message) { + return array_intersect_key($message->bodySections(), array_flip($partIds)); + } + + return []; + } + /** * Stream the raw bytes of a message or a specific MIME part without buffering. * diff --git a/tests/php/Support/FakeLiveMailService.php b/tests/php/Support/FakeLiveMailService.php index 3ca50bb..3f8d46e 100644 --- a/tests/php/Support/FakeLiveMailService.php +++ b/tests/php/Support/FakeLiveMailService.php @@ -96,6 +96,15 @@ final class FakeLiveMailService extends LiveMailService return $this->found; } + /** @var array> full section content by UID */ + public array $sections = []; + + public function messageSections(string $collection, int $uid, string ...$partIds): array + { + $this->calls[] = ['messageSections', [$collection, $uid, $partIds]]; + return array_intersect_key($this->sections[$uid] ?? [], array_flip($partIds)); + } + public function entityPatch(string $collection, array $flagsToAdd = [], array $flagsToRemove = [], int ...$uids): void { $this->calls[] = ['entityPatch', [$collection, $flagsToAdd, $flagsToRemove, $uids]]; diff --git a/tests/php/Support/FakeMessageFileStore.php b/tests/php/Support/FakeMessageFileStore.php index 233c8c7..fc853f3 100644 --- a/tests/php/Support/FakeMessageFileStore.php +++ b/tests/php/Support/FakeMessageFileStore.php @@ -36,6 +36,12 @@ final class FakeMessageFileStore extends MessageFileStore $this->files = []; } + /** Direct access for tests */ + public function content(int $uidValidity, int $uid): ?array + { + return $this->files[$uidValidity][$uid] ?? null; + } + /** @return int[] */ public function uids(int $uidValidity): array { diff --git a/tests/php/Unit/CachedServiceTest.php b/tests/php/Unit/CachedServiceTest.php index ce55bfc..4ad83d0 100644 --- a/tests/php/Unit/CachedServiceTest.php +++ b/tests/php/Unit/CachedServiceTest.php @@ -255,6 +255,81 @@ final class CachedServiceTest extends TestCase $this->assertCount(2, $this->service->entityListBulk('INBOX')); } + public function testFetchComesFromTheCache(): void + { + $this->harmonized([1 => [], 2 => ['\\Flagged']]); + $this->live->fetchBatches = []; + + $entities = array_values($this->service->entityFetchBulk( + new EntityIdentifier('imap', 'svc', 'INBOX', '2'), + )); + + $this->assertCount(1, $entities); + $this->assertSame(['flagged' => true], $entities[0]->getProperties()->getFlags()); + $this->assertSame([], $this->live->fetchBatches); + } + + public function testFetchOfAnUncachedMessageGoesToTheServerAndCaches(): void + { + $this->harmonized([1 => []]); + $this->live->flags[5] = []; + $this->live->fetchBatches = []; + + $entities = array_values($this->service->entityFetchBulk(new EntityIdentifier('imap', 'svc', 'INBOX', '5'))); + + $this->assertSame('5', (string) $entities[0]->identifier()); + $this->assertSame([[5]], $this->live->fetchBatches); + $this->assertContains(5, $this->files->uids(7)); + } + + public function testFetchFromANeverHarmonizedMailboxIsReadFromTheServer(): void + { + $this->live->flags = [3 => []]; + + $entities = array_values($this->service->entityFetchBulk(new EntityIdentifier('imap', 'svc', 'Sent', '3'))); + + $this->assertSame('3', (string) $entities[0]->identifier()); + $this->assertSame([], $this->files->uids(7), 'nothing is cached without a mailbox generation'); + } + + public function testFetchRejectsIdentifiersOfAnotherService(): void + { + $this->expectException(\InvalidArgumentException::class); + iterator_to_array($this->service->entityFetchStream(new EntityIdentifier('imap', 'other', 'INBOX', '1'))); + } + + public function testIncompleteSectionsAreCompletedOnceAndStored(): void + { + $this->harmonized([1 => []]); + $content = $this->files->content(7, 1); + $content['incomplete'] = ['1']; + $content['properties']['body'] = ['partId' => '1', 'type' => 'text/plain', 'content' => 'Hello wor']; + $this->files->write('tenant', 'svc', 'INBOX', 7, 1, $content); + $this->live->sections = [1 => ['1' => 'Hello world, the full text.']]; + + $listed = array_values(iterator_to_array($this->service->entityListStream('INBOX'), false)); + $fetched = array_values($this->service->entityFetchBulk(new EntityIdentifier('imap', 'svc', 'INBOX', '1'))); + + $this->assertSame('Hello world, the full text.', $listed[0]->getProperties()->getBodyTextPlain()); + $this->assertSame('Hello world, the full text.', $fetched[0]->getProperties()->getBodyTextPlain()); + $this->assertSame([], $this->files->content(7, 1)['incomplete']); + $this->assertCount(1, $this->calls('messageSections'), 'completed once, then served from the cache'); + } + + public function testIncompleteSectionsStayCutOffWhenTheServerDoesNotReturnThem(): void + { + $this->harmonized([1 => []]); + $content = $this->files->content(7, 1); + $content['incomplete'] = ['1']; + $content['properties']['body'] = ['partId' => '1', 'type' => 'text/plain', 'content' => 'Hello wor']; + $this->files->write('tenant', 'svc', 'INBOX', 7, 1, $content); + + $fetched = array_values($this->service->entityFetchBulk(new EntityIdentifier('imap', 'svc', 'INBOX', '1'))); + + $this->assertSame('Hello wor', $fetched[0]->getProperties()->getBodyTextPlain()); + $this->assertSame(['1'], $this->files->content(7, 1)['incomplete']); + } + /** * Harmonize INBOX with the given server UIDs and flags. * diff --git a/tests/php/Unit/MessageSectionsTest.php b/tests/php/Unit/MessageSectionsTest.php new file mode 100644 index 0000000..66c65b9 --- /dev/null +++ b/tests/php/Unit/MessageSectionsTest.php @@ -0,0 +1,59 @@ +command = $options?->toCommand(); + $structure = '(("TEXT" "PLAIN" ("CHARSET" "UTF-8") NIL NIL "BASE64" 16 1)' + . '("TEXT" "HTML" ("CHARSET" "UTF-8") NIL NIL "7BIT" 12 1) "ALTERNATIVE" ("BOUNDARY" "b") NIL NIL)'; + yield 7 => FetchMessageParser::parse( + "* 1 FETCH (UID 7 BODYSTRUCTURE {$structure} BODY[1] {16}\r\nSGVsbG8gd29ybGQ= BODY[2] {12}\r\n

Hello

)" + ); + } + }; + + $sections = $live->messageSections('INBOX', 7, '1', '2'); + + $this->assertStringContainsString('BODYSTRUCTURE', (string) $live->command); + $this->assertStringContainsString('BODY.PEEK[1]', (string) $live->command); + $this->assertStringContainsString('BODY.PEEK[2]', (string) $live->command); + $this->assertSame(['1' => 'Hello world', '2' => '

Hello

'], $sections); + } + + public function testCompleteSectionReplacesNestedContentAndClearsTheMark(): void + { + $properties = new MessageProperties([]); + $properties->jsonDeserialize(['body' => [ + 'partId' => '1', 'type' => 'multipart/alternative', 'subParts' => [ + ['partId' => '1.1', 'type' => 'text/plain', 'content' => 'Hi'], + ['partId' => '1.2', 'type' => 'text/html', 'content' => '

Hello wor'], + ], + ]]); + $properties->setIncompleteSections('1.2'); + + $this->assertTrue($properties->completeSection('1.2', '

Hello world

')); + $this->assertFalse($properties->completeSection('9', 'x')); + + $this->assertSame('

Hello world

', $properties->getBodyTextHtml()); + $this->assertSame('Hi', $properties->getBodyTextPlain()); + $this->assertSame([], $properties->getIncompleteSections()); + } +}