From b1e92e5b078148e4cbd9f37a25f62edf67c8b423 Mon Sep 17 00:00:00 2001 From: Sebastian Krupinski Date: Wed, 7 Oct 2026 22:15:34 -0400 Subject: [PATCH] feat: serve message lists from the cache Signed-off-by: Sebastian Krupinski --- lib/Providers/CachedService.php | 191 ++++++++++++++++- lib/Providers/MessageProperties.php | 1 + lib/Service/Cache/MessageQueryBuilder.php | 233 +++++++++++++++++++++ lib/Stores/MessageStore.php | 29 ++- tests/php/Support/FakeLiveMailService.php | 21 ++ tests/php/Support/FakeMessageFileStore.php | 5 + tests/php/Support/FakeMessageStore.php | 25 +++ tests/php/Unit/CachedServiceTest.php | 134 ++++++++++++ tests/php/Unit/MessageQueryBuilderTest.php | 166 +++++++++++++++ 9 files changed, 800 insertions(+), 5 deletions(-) create mode 100644 lib/Service/Cache/MessageQueryBuilder.php create mode 100644 tests/php/Unit/MessageQueryBuilderTest.php diff --git a/lib/Providers/CachedService.php b/lib/Providers/CachedService.php index 757ab3a..90f0ebf 100644 --- a/lib/Providers/CachedService.php +++ b/lib/Providers/CachedService.php @@ -22,11 +22,18 @@ use KTXF\Resource\Identifier\CollectionIdentifier; use KTXF\Resource\Identifier\EntityIdentifier; use KTXF\Resource\Identifier\EntityIdentifierInterface; use KTXF\Resource\Range\IRange; +use KTXF\Resource\Range\IRangeTally; +use KTXF\Resource\Range\RangeAnchorType; use KTXF\Resource\Sort\ISort; use KTXM\ProviderImap\Service\Cache\HarmonizationService; use KTXM\ProviderImap\Service\Cache\MessageDeltaService; +use KTXM\ProviderImap\Service\Cache\MessageIngestor; +use KTXM\ProviderImap\Service\Cache\MessageQueryBuilder; use KTXM\ProviderImap\Service\Live\LiveMailService; use KTXM\ProviderImap\Stores\MailboxStore; +use KTXM\ProviderImap\Stores\MessageFileStore; +use KTXM\ProviderImap\Stores\MessageStore; +use UnexpectedValueException; /** * IMAP mail service that serves reads from the cache. @@ -41,11 +48,18 @@ class CachedService extends ServiceBase /** Seconds after which a mailbox (or the mailbox list) is harmonized again on access */ public const FRESHNESS_WINDOW = 60; + /** Messages hydrated from the cache per meta store query */ + private const HYDRATE_BATCH_SIZE = 100; + private ?LiveMailService $liveMail = null; private ?LiveService $live = null; public function __construct( private readonly MailboxStore $mailboxStore, + private readonly MessageStore $messageStore, + private readonly MessageFileStore $fileStore, + private readonly MessageIngestor $ingestor, + private readonly MessageQueryBuilder $queries, private readonly HarmonizationService $harmonizer, private readonly MessageDeltaService $deltas, ) {} @@ -119,14 +133,42 @@ class CachedService extends ServiceBase public function entityListBulk(string|int $collection, ?IFilter $filter = null, ?ISort $sort = null, ?IRange $range = null, ?array $properties = null): array { - // served from the cache in 5.2b - return $this->live()->entityListBulk($collection, $filter, $sort, $range, $properties); + return iterator_to_array($this->entityListStream($collection, $filter, $sort, $range, $properties), true); } + /** + * List messages of a mailbox. + * + * - mailbox not harmonized yet: streamed from the server, each message cached as it passes + * - harmonized: UIDs from the meta store (filter / sort translated, range applied like the + * live service), content from message.json; a filter on body / full text has the server + * find the UIDs (IMAP SEARCH) and only the content comes from the cache + */ public function entityListStream(string|int $collection, ?IFilter $filter = null, ?ISort $sort = null, ?IRange $range = null, ?array $properties = null): Generator { - // served from the cache in 5.2b - return $this->live()->entityListStream($collection, $filter, $sort, $range, $properties); + $mailbox = (string) $collection; + $state = $this->mailboxStore->state($this->serviceId(), $mailbox); + + if (!$state['harmonizationComplete'] || $state['uidValidity'] === null) { + yield from $this->listFromServer($mailbox, $filter, $sort, $range); + return; + } + + $uidValidity = (int) $state['uidValidity']; + + if ($this->queries->needsServer($filter)) { + $uids = $this->liveMail()->entityFind($mailbox, $filter, $sort, $range); + } else { + $order = $this->queries->sort($sort); + $uids = self::applyRange( + $this->messageStore->query($this->serviceId(), $mailbox, $uidValidity, $this->queries->filter($filter), $order['sort'], $order['collation']), + $range, + ); + } + + foreach ($this->hydrate($mailbox, $uidValidity, $uids) as $entity) { + yield $entity->urn() => $entity; + } } public function entityFetchBulk(EntityIdentifierInterface ...$identifiers): array @@ -211,6 +253,142 @@ class CachedService extends ServiceBase // ── Internals ──────────────────────────────────────────────────────────── + /** + * Stream a list from the server, caching each message as it passes (cold mailbox). + */ + private function listFromServer(string $mailbox, ?IFilter $filter, ?ISort $sort, ?IRange $range): Generator + { + $uidValidity = $this->cacheGeneration($mailbox); + + foreach ($this->liveMail()->entityList($mailbox, $filter, $sort, $range) as $entity) { + if ($uidValidity !== null) { + $this->ingestQuietly($uidValidity, $entity); + } + yield $entity->urn() => $entity; + } + } + + /** + * Entities for UIDs in the given order: cached content + meta flags, with messages + * missing from the cache (or written with another schema version) fetched from the + * server and cached. + * + * @param int[] $uids + * @return Generator + */ + private function hydrate(string $mailbox, int $uidValidity, array $uids): Generator + { + foreach (array_chunk($uids, self::HYDRATE_BATCH_SIZE) as $batch) { + $metas = $this->messageStore->fetchMany($this->serviceId(), $mailbox, $uidValidity, ...$batch); + + $entities = []; + $missing = []; + foreach ($batch as $uid) { + $entity = isset($metas[$uid]) ? $this->entityFromCache($mailbox, $uidValidity, $metas[$uid]) : null; + if ($entity === null) { + $missing[] = $uid; + continue; + } + $entities[$uid] = $entity; + } + + if ($missing !== []) { + foreach ($this->liveMail()->entityFetch($mailbox, ...$missing) as $uid => $entity) { + $this->ingestQuietly($uidValidity, $entity); + $entities[$uid] = $entity; + } + } + + foreach ($batch as $uid) { + if (isset($entities[$uid])) { + yield $uid => $entities[$uid]; + } + } + } + } + + /** + * An entity from its meta document and message.json; null when the content is missing or outdated. + */ + private function entityFromCache(string $mailbox, int $uidValidity, array $meta): ?EntityResource + { + $content = $this->fileStore->read($this->tenantId(), $this->serviceId(), $mailbox, $uidValidity, (int) $meta['uid']); + if ($content === null) { + return null; + } + + try { + return $this->entityFresh()->fromCacheContent($content)->fromCacheMeta($meta); + } catch (UnexpectedValueException) { + return null; + } + } + + /** + * UIDVALIDITY to cache a not yet harmonized mailbox under; null when it cannot be cached. + */ + private function cacheGeneration(string $mailbox): ?int + { + // the mailbox document holds the change sequence, so it has to exist before ingesting + if ($this->mailboxStore->fetch($this->serviceId(), $mailbox) === null) { + try { + $this->harmonizer()->harmonizeMailboxes(); + } catch (\Throwable) { + return null; + } + if ($this->mailboxStore->fetch($this->serviceId(), $mailbox) === null) { + return null; + } + } + + $state = $this->mailboxStore->state($this->serviceId(), $mailbox); + if ($state['uidValidity'] !== null) { + return (int) $state['uidValidity']; + } + + return $this->liveMail()->mailboxFetch($mailbox)?->uidValidity(); + } + + /** + * Cache a message; a failing cache write must not fail the read that triggered it. + */ + private function ingestQuietly(int $uidValidity, EntityResource $entity): void + { + try { + $this->ingestor->ingest($this->tenantId(), $this->serviceId(), $uidValidity, $entity); + } catch (\Throwable) { + // the next harmonization caches it + } + } + + /** + * Apply a list range to sorted UIDs, as the live service does: absolute skips `position` + * messages, relative starts at the UID given as `position`. + * + * @param int[] $uids + * @return int[] + */ + private static function applyRange(array $uids, ?IRange $range): array + { + if (!$range instanceof IRangeTally) { + return array_values($uids); + } + + $tally = max(0, $range->getTally()); + if ($tally === 0) { + return []; + } + + if ($range->getAnchor() === RangeAnchorType::ABSOLUTE) { + $start = max(0, (int) $range->getPosition()); + } else { + $index = array_search((int) $range->getPosition(), $uids, true); + $start = $index === false ? 0 : $index; + } + + return array_values(array_slice($uids, $start, $tally)); + } + /** * Build a collection from its cached document; its signature is the delta signature of the mailbox. */ @@ -240,6 +418,11 @@ class CachedService extends ServiceBase return (string) $this->identifier(); } + private function tenantId(): string + { + return (string) $this->tenantIdentifier(); + } + /** * One IMAP connection per service object, shared by the LiveService and harmonization. */ diff --git a/lib/Providers/MessageProperties.php b/lib/Providers/MessageProperties.php index 5047e2b..f9bbb38 100644 --- a/lib/Providers/MessageProperties.php +++ b/lib/Providers/MessageProperties.php @@ -155,6 +155,7 @@ class MessageProperties extends MessagePropertiesMutableAbstract { 'from' => $this->data[static::PROPERTY_FROM] ?? null, 'to' => $this->data[static::PROPERTY_TO] ?? [], 'cc' => $this->data[static::PROPERTY_CC] ?? [], + 'bcc' => $this->data[static::PROPERTY_BCC] ?? [], 'urid' => $this->data[static::PROPERTY_URID] ?? null, 'inReplyTo' => $this->data[static::PROPERTY_IN_REPLY_TO] ?? null, 'references' => $this->data[static::PROPERTY_REFERENCES] ?? [], diff --git a/lib/Service/Cache/MessageQueryBuilder.php b/lib/Service/Cache/MessageQueryBuilder.php new file mode 100644 index 0000000..210e537 --- /dev/null +++ b/lib/Service/Cache/MessageQueryBuilder.php @@ -0,0 +1,233 @@ + + * SPDX-License-Identifier: AGPL-3.0-or-later + */ + +namespace KTXM\ProviderImap\Service\Cache; + +use DateTimeImmutable; +use DateTimeZone; +use KTXF\Resource\Filter\FilterComparisonOperator; +use KTXF\Resource\Filter\FilterConjunctionOperator; +use KTXF\Resource\Filter\IFilter; +use KTXF\Resource\Sort\ISort; +use MongoDB\BSON\Regex; + +/** + * Translates entity list filters and sorts into meta store queries. + * + * Mirrors LiveMailService's IMAP SEARCH / SORT translation so a cached list matches a + * live one: conditions are combined left to right with their conjunction, strings + * match as case-insensitive substrings, dates compare whole (UTC) days on `received`, + * sizes compare `size`. Conditions the cache cannot answer (body text, full text) + * make the whole filter a server filter. + */ +class MessageQueryBuilder +{ + /** Attributes that need the message body, which the meta store does not hold */ + private const SERVER_ATTRIBUTES = ['body', '*', 'all']; + + /** Address fields: matched on address and display label */ + private const ADDRESS_ATTRIBUTES = ['from', 'to', 'cc', 'bcc']; + + /** Sorts on text fields use a case-insensitive collation */ + private const TEXT_SORTS = ['subject', 'from', 'to']; + + /** + * Whether the filter has to be evaluated by the server (IMAP SEARCH). + */ + public function needsServer(?IFilter $filter): bool + { + foreach ($filter?->conditions() ?? [] as $condition) { + if (in_array($condition['attribute'] ?? '', self::SERVER_ATTRIBUTES, true)) { + return true; + } + } + + return false; + } + + /** + * Meta store filter for a list filter; [] matches everything. + * + * Only valid when needsServer() is false. + */ + public function filter(?IFilter $filter): array + { + $expression = null; + + foreach ($filter?->conditions() ?? [] as $condition) { + $operand = $this->operand($condition); + if ($operand === null) { + continue; + } + + if ($expression === null) { + $expression = $operand; + continue; + } + + $expression = (($condition['conjunction'] ?? FilterConjunctionOperator::AND) === FilterConjunctionOperator::OR) + ? ['$or' => [$expression, $operand]] + : ['$and' => [$expression, $operand]]; + } + + return $expression ?? []; + } + + /** + * Meta store sort for a list sort; always ends with uid so pages are stable. + * + * @return array{sort: array, collation: ?array} collation is set when a text field is sorted + */ + public function sort(?ISort $sort): array + { + $order = []; + $text = false; + + foreach ($sort?->conditions() ?? [] as $condition) { + $attribute = $condition['attribute'] ?? ''; + $field = match ($attribute) { + 'from' => 'from.address', + 'to' => 'to.0.address', + 'subject' => 'subject', + 'received' => 'received', + 'sent' => 'sent', + 'size' => 'size', + default => null, + }; + if ($field === null) { + continue; + } + + $order[$field] = ($condition['direction'] ?? true) ? 1 : -1; + $text = $text || in_array($attribute, self::TEXT_SORTS, true); + } + + $order['uid'] ??= 1; + + return ['sort' => $order, 'collation' => $text ? ['locale' => 'en', 'strength' => 2] : null]; + } + + /** + * @param array{attribute:string, value:mixed, comparator?:FilterComparisonOperator, conjunction?:FilterConjunctionOperator|null} $condition + */ + private function operand(array $condition): ?array + { + $attribute = $condition['attribute'] ?? ''; + $value = $condition['value'] ?? null; + $comparator = $condition['comparator'] ?? FilterComparisonOperator::EQ; + + return match (true) { + $attribute === 'subject', in_array($attribute, self::ADDRESS_ATTRIBUTES, true) => $this->stringOperand($attribute, $value, $comparator), + $attribute === 'before', $attribute === 'after' => $this->dateOperand($attribute, $value, $comparator), + $attribute === 'min', $attribute === 'max' => $this->sizeOperand($attribute, $value, $comparator), + default => null, + }; + } + + private function stringOperand(string $attribute, mixed $value, FilterComparisonOperator $comparator): ?array + { + $values = is_array($value) ? array_values($value) : [$value]; + $values = array_values(array_filter(array_map( + static fn (mixed $item): string => trim((string) $item), + $values, + ), static fn (string $item): bool => $item !== '')); + + if ($values === []) { + return null; + } + + $matches = array_map(fn (string $item): array => $this->stringMatch($attribute, $item), $values); + + return match ($comparator) { + FilterComparisonOperator::EQ, FilterComparisonOperator::LIKE, FilterComparisonOperator::IN + => count($matches) === 1 ? $matches[0] : ['$or' => $matches], + FilterComparisonOperator::NEQ, FilterComparisonOperator::NLIKE, FilterComparisonOperator::NIN + => ['$nor' => $matches], + default => null, + }; + } + + private function stringMatch(string $attribute, string $value): array + { + $regex = new Regex(preg_quote($value, '/'), 'i'); + + if ($attribute === 'subject') { + return ['subject' => $regex]; + } + + return ['$or' => [[$attribute . '.address' => $regex], [$attribute . '.label' => $regex]]]; + } + + private function dateOperand(string $attribute, mixed $value, FilterComparisonOperator $comparator): ?array + { + $day = $this->day($value); + if ($day === null) { + return null; + } + + $start = $day->format('Y-m-d\TH:i:s\Z'); + $end = $day->modify('+1 day')->format('Y-m-d\TH:i:s\Z'); + $on = ['received' => ['$gte' => $start, '$lt' => $end]]; + + return match ($comparator) { + FilterComparisonOperator::EQ => $on, + FilterComparisonOperator::NEQ => ['$nor' => [$on]], + FilterComparisonOperator::LT, FilterComparisonOperator::LTE => $attribute === 'before' ? ['received' => ['$lt' => $start]] : null, + FilterComparisonOperator::GT, FilterComparisonOperator::GTE => $attribute === 'after' ? ['received' => ['$gte' => $start]] : null, + default => null, + }; + } + + private function sizeOperand(string $attribute, mixed $value, FilterComparisonOperator $comparator): ?array + { + if (!is_int($value) && !is_numeric($value)) { + return null; + } + + $size = max(0, (int) $value); + + return match ($attribute) { + 'min' => match ($comparator) { + FilterComparisonOperator::EQ, FilterComparisonOperator::GTE => ['size' => ['$gte' => $size]], + FilterComparisonOperator::GT => ['size' => ['$gt' => $size]], + FilterComparisonOperator::LT => ['size' => ['$lt' => $size]], + FilterComparisonOperator::LTE => ['size' => ['$lte' => $size]], + FilterComparisonOperator::NEQ => ['size' => ['$ne' => $size]], + default => null, + }, + 'max' => match ($comparator) { + FilterComparisonOperator::EQ, FilterComparisonOperator::LTE => ['size' => ['$lte' => $size]], + FilterComparisonOperator::LT => ['size' => ['$lt' => $size]], + FilterComparisonOperator::GT => ['size' => ['$gt' => $size]], + FilterComparisonOperator::GTE => ['size' => ['$gte' => $size]], + FilterComparisonOperator::NEQ => ['size' => ['$ne' => $size]], + default => null, + }, + default => null, + }; + } + + /** + * Start of the (UTC) day a filter value falls on. + */ + private function day(mixed $value): ?DateTimeImmutable + { + if ($value === null || $value === '') { + return null; + } + + try { + $date = $value instanceof DateTimeImmutable ? $value : new DateTimeImmutable(trim((string) $value)); + } catch (\Exception) { + return null; + } + + return new DateTimeImmutable($date->format('Y-m-d'), new DateTimeZone('UTC')); + } +} diff --git a/lib/Stores/MessageStore.php b/lib/Stores/MessageStore.php index 0a6d3e4..f993ebd 100644 --- a/lib/Stores/MessageStore.php +++ b/lib/Stores/MessageStore.php @@ -34,7 +34,7 @@ class MessageStore /** Fields dropped when a message becomes a tombstone */ private const TOMBSTONE_UNSET = [ 'created' => '', 'received' => '', 'sent' => '', 'size' => '', - 'subject' => '', 'from' => '', 'to' => '', 'cc' => '', + 'subject' => '', 'from' => '', 'to' => '', 'cc' => '', 'bcc' => '', 'urid' => '', 'inReplyTo' => '', 'references' => '', 'flags' => '', 'hasAttachments' => '', 'preview' => '', 'blobs' => '', ]; @@ -179,6 +179,33 @@ class MessageStore return $uids; } + /** + * UIDs of the cached messages matching a filter, in sort order (tombstones excluded). + * + * @param array $filter meta store filter (see MessageQueryBuilder::filter()) + * @param array $sort meta store sort (see MessageQueryBuilder::sort()) + * @param array|null $collation collation for text sorts + * @return int[] + */ + public function query(string $serviceId, string $mailbox, int $uidValidity, array $filter = [], array $sort = ['uid' => 1], ?array $collation = null): array + { + $match = ['sid' => $serviceId, 'mailbox' => $mailbox, 'uidValidity' => $uidValidity, ...self::LIVE]; + if ($filter !== []) { + $match = ['$and' => [$match, $filter]]; + } + + $options = ['projection' => ['uid' => 1], 'sort' => $sort]; + if ($collation !== null) { + $options['collation'] = $collation; + } + + $uids = []; + foreach ($this->dataStore->selectCollection(self::COLLECTION_NAME)->find($match, $options) as $document) { + $uids[] = (int) $document['uid']; + } + return $uids; + } + /** * List the flags of every message cached for a mailbox. * diff --git a/tests/php/Support/FakeLiveMailService.php b/tests/php/Support/FakeLiveMailService.php index 54e9bae..3ca50bb 100644 --- a/tests/php/Support/FakeLiveMailService.php +++ b/tests/php/Support/FakeLiveMailService.php @@ -6,6 +6,7 @@ namespace KTXT\ProviderImap\Tests\Support; use Generator; use KTXF\Resource\Filter\IFilter; +use KTXF\Resource\Range\IRange; use KTXF\Resource\Sort\ISort; use KTXM\ProviderImap\Client\Mailbox; use KTXM\ProviderImap\Client\Protocol\Command\Argument\FetchOptions; @@ -75,6 +76,26 @@ final class FakeLiveMailService extends LiveMailService } } + /** + * Every message of the mailbox in UID order; filter / sort / range are not evaluated. + */ + public function messageList(string $collection, ?IFilter $filter = null, ?ISort $sort = null, ?IRange $range = null): Generator + { + $this->calls[] = ['messageList', [$collection]]; + $uids = array_keys($this->flags); + sort($uids); + yield from $this->messageFetch($collection, null, ...$uids); + } + + /** @var int[] UIDs entityFind() returns */ + public array $found = []; + + public function entityFind(string $collection, ?IFilter $filter = null, ?ISort $sort = null, ?IRange $range = null): array + { + $this->calls[] = ['entityFind', [$collection, $filter]]; + return $this->found; + } + public function entityPatch(string $collection, array $flagsToAdd = [], array $flagsToRemove = [], int ...$uids): void { $this->calls[] = ['entityPatch', [$collection, $flagsToAdd, $flagsToRemove, $uids]]; diff --git a/tests/php/Support/FakeMessageFileStore.php b/tests/php/Support/FakeMessageFileStore.php index 657f869..233c8c7 100644 --- a/tests/php/Support/FakeMessageFileStore.php +++ b/tests/php/Support/FakeMessageFileStore.php @@ -19,6 +19,11 @@ final class FakeMessageFileStore extends MessageFileStore $this->files[$uidValidity][$uid] = $content; } + public function read(string $tenantId, string $serviceId, string $mailbox, int $uidValidity, int $uid): ?array + { + return $this->files[$uidValidity][$uid] ?? null; + } + public function delete(string $tenantId, string $serviceId, string $mailbox, int $uidValidity, int ...$uids): void { foreach ($uids as $uid) { diff --git a/tests/php/Support/FakeMessageStore.php b/tests/php/Support/FakeMessageStore.php index cbe9e6f..876b80e 100644 --- a/tests/php/Support/FakeMessageStore.php +++ b/tests/php/Support/FakeMessageStore.php @@ -20,6 +20,7 @@ final class FakeMessageStore extends MessageStore $meta = $entity->toCacheMeta(); $existing = $this->documents[$uidValidity][$meta['uid']] ?? null; $this->documents[$uidValidity][$meta['uid']] = [ + 'meta' => $meta, 'flags' => $meta['flags'], 'addedSeq' => $existing['addedSeq'] ?? $seq, 'modSeq' => $seq, @@ -28,6 +29,30 @@ final class FakeMessageStore extends MessageStore ksort($this->documents[$uidValidity]); } + /** Last query() arguments, for assertions */ + public ?array $lastQuery = null; + + /** + * Live UIDs in ascending order; the filter / sort translation is covered by MessageQueryBuilderTest. + */ + public function query(string $serviceId, string $mailbox, int $uidValidity, array $filter = [], array $sort = ['uid' => 1], ?array $collation = null): array + { + $this->lastQuery = [$filter, $sort, $collation]; + return array_keys($this->flags($serviceId, $mailbox, $uidValidity)); + } + + public function fetchMany(string $serviceId, string $mailbox, int $uidValidity, int ...$uids): array + { + $list = []; + foreach ($uids as $uid) { + $document = $this->documents[$uidValidity][$uid] ?? null; + if ($document !== null && $document['removedSeq'] === null) { + $list[$uid] = [...$document['meta'], 'uid' => $uid, 'flags' => $document['flags']]; + } + } + return $list; + } + public function flags(string $serviceId, string $mailbox, int $uidValidity): array { $flags = []; diff --git a/tests/php/Unit/CachedServiceTest.php b/tests/php/Unit/CachedServiceTest.php index 743a84f..ce55bfc 100644 --- a/tests/php/Unit/CachedServiceTest.php +++ b/tests/php/Unit/CachedServiceTest.php @@ -12,8 +12,12 @@ use KTXM\ProviderImap\Providers\MessageProperties; use KTXM\ProviderImap\Service\Cache\HarmonizationService; use KTXM\ProviderImap\Service\Cache\MessageDeltaService; use KTXM\ProviderImap\Service\Cache\MessageIngestor; +use KTXM\ProviderImap\Service\Cache\MessageQueryBuilder; use KTXM\ProviderImap\Service\Live\LiveMailService; use KTXF\Resource\Identifier\EntityIdentifier; +use KTXF\Resource\Range\RangeAnchorType; +use KTXF\Resource\Range\RangeType; +use KTXF\Resource\Sort\Sort; use KTXT\ProviderImap\Tests\Support\FakeLiveMailService; use KTXT\ProviderImap\Tests\Support\FakeMailboxStore; use KTXT\ProviderImap\Tests\Support\FakeMessageFileStore; @@ -23,6 +27,8 @@ use PHPUnit\Framework\TestCase; final class CachedServiceTest extends TestCase { private FakeMailboxStore $mailboxes; + private FakeMessageStore $messages; + private FakeMessageFileStore $files; private FakeLiveMailService $live; private CachedService $service; @@ -40,8 +46,14 @@ final class CachedServiceTest extends TestCase new MessageIngestor($files, $messages, $this->mailboxes), ); + $this->messages = $messages; + $this->files = $files; $this->service = new CachedServiceTestDouble( $this->mailboxes, + $messages, + $files, + new MessageIngestor($files, $messages, $this->mailboxes), + new MessageQueryBuilder(), $harmonizer, new MessageDeltaService($this->mailboxes, $messages), ); @@ -152,6 +164,128 @@ final class CachedServiceTest extends TestCase $this->assertSame([5], $uids); } + public function testListOfANotHarmonizedMailboxStreamsFromTheServerAndCaches(): void + { + $this->live->flags = [1 => [], 2 => ['\\Seen']]; + + $uids = $this->listedUids($this->service->entityListStream('INBOX')); + + $this->assertSame([1, 2], $uids); + $this->assertCount(1, $this->calls('messageList')); + $this->assertSame([1, 2], $this->files->uids(7)); + $this->assertSame([1 => [], 2 => ['seen']], $this->messages->flags('svc', 'INBOX', 7)); + } + + public function testListOfAHarmonizedMailboxComesFromTheCache(): void + { + $this->harmonized([1 => [], 2 => ['\\Seen'], 3 => []]); + + $entities = iterator_to_array($this->service->entityListStream('INBOX'), false); + + $this->assertSame([1, 2, 3], array_map(static fn ($entity): int => (int) $entity->identifier(), $entities)); + $this->assertSame([], $this->calls('messageList')); + $this->assertSame(['seen' => true], $entities[1]->getProperties()->getFlags()); + $this->assertSame('INBOX', $entities[1]->collection()); + } + + public function testListRangeIsAppliedLikeTheLiveService(): void + { + $this->harmonized([1 => [], 2 => [], 3 => [], 4 => []]); + + $absolute = $this->service->entityListRange(RangeType::TALLY); + $absolute->setAnchor(RangeAnchorType::ABSOLUTE); + $absolute->setPosition(1); + $absolute->setTally(2); + + $relative = $this->service->entityListRange(RangeType::TALLY); + $relative->setAnchor(RangeAnchorType::RELATIVE); + $relative->setPosition(3); + $relative->setTally(2); + + $this->assertSame([2, 3], $this->listedUids($this->service->entityListStream('INBOX', null, null, $absolute))); + $this->assertSame([3, 4], $this->listedUids($this->service->entityListStream('INBOX', null, null, $relative))); + } + + public function testMessagesMissingFromTheContentStoreAreFetchedAndCached(): void + { + $this->harmonized([1 => [], 2 => []]); + $this->files->delete('tenant', 'svc', 'INBOX', 7, 2); + $this->live->fetchBatches = []; + + $this->assertSame([1, 2], $this->listedUids($this->service->entityListStream('INBOX'))); + $this->assertSame([[2]], $this->live->fetchBatches); + $this->assertSame([1, 2], $this->files->uids(7)); + } + + public function testBodyFilterHasTheServerFindTheUids(): void + { + $this->harmonized([1 => [], 2 => [], 3 => []]); + $this->live->found = [3, 1]; + $this->live->fetchBatches = []; + + $filter = $this->service->entityListFilter(); + $filter->condition('body', 'invoice'); + + $this->assertSame([3, 1], $this->listedUids($this->service->entityListStream('INBOX', $filter))); + $this->assertCount(1, $this->calls('entityFind')); + $this->assertSame([], $this->live->fetchBatches, 'content comes from the cache'); + } + + public function testOtherFiltersAreTranslatedForTheMetaStore(): void + { + $this->harmonized([1 => []]); + + $filter = $this->service->entityListFilter(); + $filter->condition('subject', 'report'); + // keyed attributes: entityListSort() passes a plain list, which Sort::condition() ignores (existing bug) + $sort = new Sort(['received' => true]); + $sort->condition('received', false); + + iterator_to_array($this->service->entityListStream('INBOX', $filter, $sort)); + + $this->assertSame([], $this->calls('entityFind')); + $this->assertArrayHasKey('subject', $this->messages->lastQuery[0]); + $this->assertSame(['received' => -1, 'uid' => 1], $this->messages->lastQuery[1]); + } + + public function testListBulkIsTheStreamCollected(): void + { + $this->harmonized([1 => [], 2 => []]); + + $this->assertCount(2, $this->service->entityListBulk('INBOX')); + } + + /** + * Harmonize INBOX with the given server UIDs and flags. + * + * @param array> $flags + */ + private function harmonized(array $flags): void + { + $this->live->flags = $flags; + $this->service->entityDelta('INBOX', ''); + } + + /** + * @return int[] + */ + private function listedUids(iterable $entities): array + { + $uids = []; + foreach ($entities as $entity) { + $uids[] = (int) $entity->identifier(); + } + return $uids; + } + + /** + * @return list + */ + private function calls(string $method): array + { + return array_values(array_filter($this->live->calls, static fn (array $call): bool => $call[0] === $method)); + } + /** * @return list */ diff --git a/tests/php/Unit/MessageQueryBuilderTest.php b/tests/php/Unit/MessageQueryBuilderTest.php new file mode 100644 index 0000000..684a89c --- /dev/null +++ b/tests/php/Unit/MessageQueryBuilderTest.php @@ -0,0 +1,166 @@ +builder = new MessageQueryBuilder(); + } + + public function testNoFilterMatchesEverything(): void + { + $this->assertSame([], $this->builder->filter(null)); + $this->assertFalse($this->builder->needsServer(null)); + } + + public function testBodyAndFullTextNeedTheServer(): void + { + $this->assertTrue($this->builder->needsServer($this->filter(['body', 'invoice']))); + $this->assertTrue($this->builder->needsServer($this->filter(['*', 'invoice']))); + $this->assertFalse($this->builder->needsServer($this->filter(['subject', 'invoice']))); + } + + public function testSubjectIsACaseInsensitiveSubstring(): void + { + $filter = $this->builder->filter($this->filter(['subject', 'Q3 (draft)'])); + + $this->assertEquals(['subject' => new Regex('Q3 \\(draft\\)', 'i')], $filter); + } + + public function testAddressesMatchAddressOrLabel(): void + { + $filter = $this->builder->filter($this->filter(['from', 'alice'])); + + $this->assertEquals(['$or' => [ + ['from.address' => new Regex('alice', 'i')], + ['from.label' => new Regex('alice', 'i')], + ]], $filter); + } + + public function testNegationAndMultipleValues(): void + { + $not = $this->builder->filter($this->filter(['subject', 'spam', FilterComparisonOperator::NLIKE])); + $any = $this->builder->filter($this->filter(['subject', ['a', 'b'], FilterComparisonOperator::IN])); + + $this->assertEquals(['$nor' => [['subject' => new Regex('spam', 'i')]]], $not); + $this->assertEquals(['$or' => [['subject' => new Regex('a', 'i')], ['subject' => new Regex('b', 'i')]]], $any); + } + + public function testConditionsCombineLeftToRightWithTheirConjunction(): void + { + $filter = $this->builder->filter($this->filter( + ['subject', 'a'], + ['subject', 'b', null, FilterConjunctionOperator::OR], + ['min', 100, FilterComparisonOperator::GTE, FilterConjunctionOperator::AND], + )); + + $this->assertEquals(['$and' => [ + ['$or' => [['subject' => new Regex('a', 'i')], ['subject' => new Regex('b', 'i')]]], + ['size' => ['$gte' => 100]], + ]], $filter); + } + + public function testDatesCompareWholeUtcDaysOnReceived(): void + { + $before = $this->builder->filter($this->filter(['before', '2026-10-05T18:00:00+02:00', FilterComparisonOperator::LT])); + $after = $this->builder->filter($this->filter(['after', '2026-10-05', FilterComparisonOperator::GTE])); + $on = $this->builder->filter($this->filter(['after', '2026-10-05', FilterComparisonOperator::EQ])); + + $this->assertSame(['received' => ['$lt' => '2026-10-05T00:00:00Z']], $before); + $this->assertSame(['received' => ['$gte' => '2026-10-05T00:00:00Z']], $after); + $this->assertSame(['received' => ['$gte' => '2026-10-05T00:00:00Z', '$lt' => '2026-10-06T00:00:00Z']], $on); + } + + public function testSizes(): void + { + $this->assertSame(['size' => ['$lte' => 500]], $this->builder->filter($this->filter(['max', 500, FilterComparisonOperator::LTE]))); + $this->assertSame(['size' => ['$gt' => 10]], $this->builder->filter($this->filter(['min', 10, FilterComparisonOperator::GT]))); + } + + public function testUnknownAttributesAndEmptyValuesAreIgnored(): void + { + $this->assertSame([], $this->builder->filter($this->filter(['unknown', 'x'], ['subject', ' ']))); + } + + public function testSortMapsFieldsAndEndsWithUid(): void + { + $sort = new Sort(['received' => true, 'subject' => true, 'to' => true]); + $sort->condition('received', false); + + $this->assertSame(['sort' => ['received' => -1, 'uid' => 1], 'collation' => null], $this->builder->sort($sort)); + + $sort->condition('to', true); + $result = $this->builder->sort($sort); + $this->assertSame(['received' => -1, 'to.0.address' => 1, 'uid' => 1], $result['sort']); + $this->assertSame(['locale' => 'en', 'strength' => 2], $result['collation']); + } + + public function testNoSortIsUidOrder(): void + { + $this->assertSame(['sort' => ['uid' => 1], 'collation' => null], $this->builder->sort(null)); + } + + public function testMetaStoreQueryExcludesTombstonesAndAppliesSortAndCollation(): void + { + $collection = $this->createMock(Collection::class); + $collection->expects($this->once()) + ->method('find') + ->with( + ['$and' => [ + ['sid' => 'svc', 'mailbox' => 'INBOX', 'uidValidity' => 7, 'removedSeq' => null], + ['size' => ['$gt' => 1]], + ]], + ['projection' => ['uid' => 1], 'sort' => ['subject' => 1, 'uid' => 1], 'collation' => ['locale' => 'en', 'strength' => 2]], + ) + ->willReturn(new Cursor(new ArrayIterator([['uid' => 4], ['uid' => 2]]))); + $dataStore = $this->createStub(DataStore::class); + $dataStore->method('selectCollection')->willReturn($collection); + + $uids = (new MessageStore($dataStore))->query('svc', 'INBOX', 7, ['size' => ['$gt' => 1]], ['subject' => 1, 'uid' => 1], ['locale' => 'en', 'strength' => 2]); + + $this->assertSame([4, 2], $uids); + } + + /** + * A filter holding the given conditions as-is (Filter itself keeps one condition per attribute). + * + * @param array{0: string, 1: mixed, 2?: ?FilterComparisonOperator, 3?: ?FilterConjunctionOperator} ...$conditions + */ + private function filter(array ...$conditions): IFilter + { + $list = array_map(static fn (array $condition): array => [ + 'attribute' => $condition[0], + 'value' => $condition[1], + 'comparator' => $condition[2] ?? FilterComparisonOperator::EQ, + 'conjunction' => $condition[3] ?? null, + ], $conditions); + + return new class($list) implements IFilter { + public function __construct(private readonly array $list) {} + public function attributes(): array { return []; } + public function comparators(): string { return ''; } + public function conjunctions(): string { return ''; } + public function condition(string $property, mixed $value, ?FilterComparisonOperator $comparator = null, ?FilterConjunctionOperator $conjunction = null): void {} + public function conditions(): array { return $this->list; } + }; + } +}