feat: serve message lists from the cache

Signed-off-by: Sebastian Krupinski <krupinski01@gmail.com>

lib/Providers/CachedService.php

@@ -22,11 +22,18 @@ use KTXF\Resource\Identifier\CollectionIdentifier;
 use KTXF\Resource\Identifier\EntityIdentifier;
 use KTXF\Resource\Identifier\EntityIdentifierInterface;
 use KTXF\Resource\Range\IRange;
+use KTXF\Resource\Range\IRangeTally;
+use KTXF\Resource\Range\RangeAnchorType;
 use KTXF\Resource\Sort\ISort;
 use KTXM\ProviderImap\Service\Cache\HarmonizationService;
 use KTXM\ProviderImap\Service\Cache\MessageDeltaService;
+use KTXM\ProviderImap\Service\Cache\MessageIngestor;
+use KTXM\ProviderImap\Service\Cache\MessageQueryBuilder;
 use KTXM\ProviderImap\Service\Live\LiveMailService;
 use KTXM\ProviderImap\Stores\MailboxStore;
+use KTXM\ProviderImap\Stores\MessageFileStore;
+use KTXM\ProviderImap\Stores\MessageStore;
+use UnexpectedValueException;
 
 /**
  * IMAP mail service that serves reads from the cache.
@@ -41,11 +48,18 @@ class CachedService extends ServiceBase
     /** Seconds after which a mailbox (or the mailbox list) is harmonized again on access */
     public const FRESHNESS_WINDOW = 60;
 
+    /** Messages hydrated from the cache per meta store query */
+    private const HYDRATE_BATCH_SIZE = 100;
+
     private ?LiveMailService $liveMail = null;
     private ?LiveService $live = null;
 
     public function __construct(
         private readonly MailboxStore $mailboxStore,
+        private readonly MessageStore $messageStore,
+        private readonly MessageFileStore $fileStore,
+        private readonly MessageIngestor $ingestor,
+        private readonly MessageQueryBuilder $queries,
         private readonly HarmonizationService $harmonizer,
         private readonly MessageDeltaService $deltas,
     ) {}
@@ -119,14 +133,42 @@ class CachedService extends ServiceBase
 
     public function entityListBulk(string|int $collection, ?IFilter $filter = null, ?ISort $sort = null, ?IRange $range = null, ?array $properties = null): array
     {
-        // served from the cache in 5.2b
-        return $this->live()->entityListBulk($collection, $filter, $sort, $range, $properties);
+        return iterator_to_array($this->entityListStream($collection, $filter, $sort, $range, $properties), true);
     }
 
+    /**
+     * List messages of a mailbox.
+     *
+     * - mailbox not harmonized yet: streamed from the server, each message cached as it passes
+     * - harmonized: UIDs from the meta store (filter / sort translated, range applied like the
+     *   live service), content from message.json; a filter on body / full text has the server
+     *   find the UIDs (IMAP SEARCH) and only the content comes from the cache
+     */
     public function entityListStream(string|int $collection, ?IFilter $filter = null, ?ISort $sort = null, ?IRange $range = null, ?array $properties = null): Generator
     {
-        // served from the cache in 5.2b
-        return $this->live()->entityListStream($collection, $filter, $sort, $range, $properties);
+        $mailbox = (string) $collection;
+        $state = $this->mailboxStore->state($this->serviceId(), $mailbox);
+
+        if (!$state['harmonizationComplete'] || $state['uidValidity'] === null) {
+            yield from $this->listFromServer($mailbox, $filter, $sort, $range);
+            return;
+        }
+
+        $uidValidity = (int) $state['uidValidity'];
+
+        if ($this->queries->needsServer($filter)) {
+            $uids = $this->liveMail()->entityFind($mailbox, $filter, $sort, $range);
+        } else {
+            $order = $this->queries->sort($sort);
+            $uids = self::applyRange(
+                $this->messageStore->query($this->serviceId(), $mailbox, $uidValidity, $this->queries->filter($filter), $order['sort'], $order['collation']),
+                $range,
+            );
+        }
+
+        foreach ($this->hydrate($mailbox, $uidValidity, $uids) as $entity) {
+            yield $entity->urn() => $entity;
+        }
     }
 
     public function entityFetchBulk(EntityIdentifierInterface ...$identifiers): array
@@ -211,6 +253,142 @@ class CachedService extends ServiceBase
 
     // ── Internals ────────────────────────────────────────────────────────────
 
+    /**
+     * Stream a list from the server, caching each message as it passes (cold mailbox).
+     */
+    private function listFromServer(string $mailbox, ?IFilter $filter, ?ISort $sort, ?IRange $range): Generator
+    {
+        $uidValidity = $this->cacheGeneration($mailbox);
+
+        foreach ($this->liveMail()->entityList($mailbox, $filter, $sort, $range) as $entity) {
+            if ($uidValidity !== null) {
+                $this->ingestQuietly($uidValidity, $entity);
+            }
+            yield $entity->urn() => $entity;
+        }
+    }
+
+    /**
+     * Entities for UIDs in the given order: cached content + meta flags, with messages
+     * missing from the cache (or written with another schema version) fetched from the
+     * server and cached.
+     *
+     * @param int[] $uids
+     * @return Generator<int, EntityResource>
+     */
+    private function hydrate(string $mailbox, int $uidValidity, array $uids): Generator
+    {
+        foreach (array_chunk($uids, self::HYDRATE_BATCH_SIZE) as $batch) {
+            $metas = $this->messageStore->fetchMany($this->serviceId(), $mailbox, $uidValidity, ...$batch);
+
+            $entities = [];
+            $missing = [];
+            foreach ($batch as $uid) {
+                $entity = isset($metas[$uid]) ? $this->entityFromCache($mailbox, $uidValidity, $metas[$uid]) : null;
+                if ($entity === null) {
+                    $missing[] = $uid;
+                    continue;
+                }
+                $entities[$uid] = $entity;
+            }
+
+            if ($missing !== []) {
+                foreach ($this->liveMail()->entityFetch($mailbox, ...$missing) as $uid => $entity) {
+                    $this->ingestQuietly($uidValidity, $entity);
+                    $entities[$uid] = $entity;
+                }
+            }
+
+            foreach ($batch as $uid) {
+                if (isset($entities[$uid])) {
+                    yield $uid => $entities[$uid];
+                }
+            }
+        }
+    }
+
+    /**
+     * An entity from its meta document and message.json; null when the content is missing or outdated.
+     */
+    private function entityFromCache(string $mailbox, int $uidValidity, array $meta): ?EntityResource
+    {
+        $content = $this->fileStore->read($this->tenantId(), $this->serviceId(), $mailbox, $uidValidity, (int) $meta['uid']);
+        if ($content === null) {
+            return null;
+        }
+
+        try {
+            return $this->entityFresh()->fromCacheContent($content)->fromCacheMeta($meta);
+        } catch (UnexpectedValueException) {
+            return null;
+        }
+    }
+
+    /**
+     * UIDVALIDITY to cache a not yet harmonized mailbox under; null when it cannot be cached.
+     */
+    private function cacheGeneration(string $mailbox): ?int
+    {
+        // the mailbox document holds the change sequence, so it has to exist before ingesting
+        if ($this->mailboxStore->fetch($this->serviceId(), $mailbox) === null) {
+            try {
+                $this->harmonizer()->harmonizeMailboxes();
+            } catch (\Throwable) {
+                return null;
+            }
+            if ($this->mailboxStore->fetch($this->serviceId(), $mailbox) === null) {
+                return null;
+            }
+        }
+
+        $state = $this->mailboxStore->state($this->serviceId(), $mailbox);
+        if ($state['uidValidity'] !== null) {
+            return (int) $state['uidValidity'];
+        }
+
+        return $this->liveMail()->mailboxFetch($mailbox)?->uidValidity();
+    }
+
+    /**
+     * Cache a message; a failing cache write must not fail the read that triggered it.
+     */
+    private function ingestQuietly(int $uidValidity, EntityResource $entity): void
+    {
+        try {
+            $this->ingestor->ingest($this->tenantId(), $this->serviceId(), $uidValidity, $entity);
+        } catch (\Throwable) {
+            // the next harmonization caches it
+        }
+    }
+
+    /**
+     * Apply a list range to sorted UIDs, as the live service does: absolute skips `position`
+     * messages, relative starts at the UID given as `position`.
+     *
+     * @param int[] $uids
+     * @return int[]
+     */
+    private static function applyRange(array $uids, ?IRange $range): array
+    {
+        if (!$range instanceof IRangeTally) {
+            return array_values($uids);
+        }
+
+        $tally = max(0, $range->getTally());
+        if ($tally === 0) {
+            return [];
+        }
+
+        if ($range->getAnchor() === RangeAnchorType::ABSOLUTE) {
+            $start = max(0, (int) $range->getPosition());
+        } else {
+            $index = array_search((int) $range->getPosition(), $uids, true);
+            $start = $index === false ? 0 : $index;
+        }
+
+        return array_values(array_slice($uids, $start, $tally));
+    }
+
     /**
      * Build a collection from its cached document; its signature is the delta signature of the mailbox.
      */
@@ -240,6 +418,11 @@ class CachedService extends ServiceBase
         return (string) $this->identifier();
     }
 
+    private function tenantId(): string
+    {
+        return (string) $this->tenantIdentifier();
+    }
+
     /**
      * One IMAP connection per service object, shared by the LiveService and harmonization.
      */

lib/Providers/MessageProperties.php

@@ -155,6 +155,7 @@ class MessageProperties extends MessagePropertiesMutableAbstract {
             'from'           => $this->data[static::PROPERTY_FROM] ?? null,
             'to'             => $this->data[static::PROPERTY_TO] ?? [],
             'cc'             => $this->data[static::PROPERTY_CC] ?? [],
+            'bcc'            => $this->data[static::PROPERTY_BCC] ?? [],
             'urid'           => $this->data[static::PROPERTY_URID] ?? null,
             'inReplyTo'      => $this->data[static::PROPERTY_IN_REPLY_TO] ?? null,
             'references'     => $this->data[static::PROPERTY_REFERENCES] ?? [],

lib/Service/Cache/MessageQueryBuilder.php

@@ -0,0 +1,233 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: Sebastian Krupinski <krupinski01@gmail.com>
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace KTXM\ProviderImap\Service\Cache;
+
+use DateTimeImmutable;
+use DateTimeZone;
+use KTXF\Resource\Filter\FilterComparisonOperator;
+use KTXF\Resource\Filter\FilterConjunctionOperator;
+use KTXF\Resource\Filter\IFilter;
+use KTXF\Resource\Sort\ISort;
+use MongoDB\BSON\Regex;
+
+/**
+ * Translates entity list filters and sorts into meta store queries.
+ *
+ * Mirrors LiveMailService's IMAP SEARCH / SORT translation so a cached list matches a
+ * live one: conditions are combined left to right with their conjunction, strings
+ * match as case-insensitive substrings, dates compare whole (UTC) days on `received`,
+ * sizes compare `size`. Conditions the cache cannot answer (body text, full text)
+ * make the whole filter a server filter.
+ */
+class MessageQueryBuilder
+{
+    /** Attributes that need the message body, which the meta store does not hold */
+    private const SERVER_ATTRIBUTES = ['body', '*', 'all'];
+
+    /** Address fields: matched on address and display label */
+    private const ADDRESS_ATTRIBUTES = ['from', 'to', 'cc', 'bcc'];
+
+    /** Sorts on text fields use a case-insensitive collation */
+    private const TEXT_SORTS = ['subject', 'from', 'to'];
+
+    /**
+     * Whether the filter has to be evaluated by the server (IMAP SEARCH).
+     */
+    public function needsServer(?IFilter $filter): bool
+    {
+        foreach ($filter?->conditions() ?? [] as $condition) {
+            if (in_array($condition['attribute'] ?? '', self::SERVER_ATTRIBUTES, true)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Meta store filter for a list filter; [] matches everything.
+     *
+     * Only valid when needsServer() is false.
+     */
+    public function filter(?IFilter $filter): array
+    {
+        $expression = null;
+
+        foreach ($filter?->conditions() ?? [] as $condition) {
+            $operand = $this->operand($condition);
+            if ($operand === null) {
+                continue;
+            }
+
+            if ($expression === null) {
+                $expression = $operand;
+                continue;
+            }
+
+            $expression = (($condition['conjunction'] ?? FilterConjunctionOperator::AND) === FilterConjunctionOperator::OR)
+                ? ['$or' => [$expression, $operand]]
+                : ['$and' => [$expression, $operand]];
+        }
+
+        return $expression ?? [];
+    }
+
+    /**
+     * Meta store sort for a list sort; always ends with uid so pages are stable.
+     *
+     * @return array{sort: array<string, int>, collation: ?array} collation is set when a text field is sorted
+     */
+    public function sort(?ISort $sort): array
+    {
+        $order = [];
+        $text = false;
+
+        foreach ($sort?->conditions() ?? [] as $condition) {
+            $attribute = $condition['attribute'] ?? '';
+            $field = match ($attribute) {
+                'from' => 'from.address',
+                'to' => 'to.0.address',
+                'subject' => 'subject',
+                'received' => 'received',
+                'sent' => 'sent',
+                'size' => 'size',
+                default => null,
+            };
+            if ($field === null) {
+                continue;
+            }
+
+            $order[$field] = ($condition['direction'] ?? true) ? 1 : -1;
+            $text = $text || in_array($attribute, self::TEXT_SORTS, true);
+        }
+
+        $order['uid'] ??= 1;
+
+        return ['sort' => $order, 'collation' => $text ? ['locale' => 'en', 'strength' => 2] : null];
+    }
+
+    /**
+     * @param array{attribute:string, value:mixed, comparator?:FilterComparisonOperator, conjunction?:FilterConjunctionOperator|null} $condition
+     */
+    private function operand(array $condition): ?array
+    {
+        $attribute = $condition['attribute'] ?? '';
+        $value = $condition['value'] ?? null;
+        $comparator = $condition['comparator'] ?? FilterComparisonOperator::EQ;
+
+        return match (true) {
+            $attribute === 'subject', in_array($attribute, self::ADDRESS_ATTRIBUTES, true) => $this->stringOperand($attribute, $value, $comparator),
+            $attribute === 'before', $attribute === 'after' => $this->dateOperand($attribute, $value, $comparator),
+            $attribute === 'min', $attribute === 'max' => $this->sizeOperand($attribute, $value, $comparator),
+            default => null,
+        };
+    }
+
+    private function stringOperand(string $attribute, mixed $value, FilterComparisonOperator $comparator): ?array
+    {
+        $values = is_array($value) ? array_values($value) : [$value];
+        $values = array_values(array_filter(array_map(
+            static fn (mixed $item): string => trim((string) $item),
+            $values,
+        ), static fn (string $item): bool => $item !== ''));
+
+        if ($values === []) {
+            return null;
+        }
+
+        $matches = array_map(fn (string $item): array => $this->stringMatch($attribute, $item), $values);
+
+        return match ($comparator) {
+            FilterComparisonOperator::EQ, FilterComparisonOperator::LIKE, FilterComparisonOperator::IN
+                => count($matches) === 1 ? $matches[0] : ['$or' => $matches],
+            FilterComparisonOperator::NEQ, FilterComparisonOperator::NLIKE, FilterComparisonOperator::NIN
+                => ['$nor' => $matches],
+            default => null,
+        };
+    }
+
+    private function stringMatch(string $attribute, string $value): array
+    {
+        $regex = new Regex(preg_quote($value, '/'), 'i');
+
+        if ($attribute === 'subject') {
+            return ['subject' => $regex];
+        }
+
+        return ['$or' => [[$attribute . '.address' => $regex], [$attribute . '.label' => $regex]]];
+    }
+
+    private function dateOperand(string $attribute, mixed $value, FilterComparisonOperator $comparator): ?array
+    {
+        $day = $this->day($value);
+        if ($day === null) {
+            return null;
+        }
+
+        $start = $day->format('Y-m-d\TH:i:s\Z');
+        $end = $day->modify('+1 day')->format('Y-m-d\TH:i:s\Z');
+        $on = ['received' => ['$gte' => $start, '$lt' => $end]];
+
+        return match ($comparator) {
+            FilterComparisonOperator::EQ => $on,
+            FilterComparisonOperator::NEQ => ['$nor' => [$on]],
+            FilterComparisonOperator::LT, FilterComparisonOperator::LTE => $attribute === 'before' ? ['received' => ['$lt' => $start]] : null,
+            FilterComparisonOperator::GT, FilterComparisonOperator::GTE => $attribute === 'after' ? ['received' => ['$gte' => $start]] : null,
+            default => null,
+        };
+    }
+
+    private function sizeOperand(string $attribute, mixed $value, FilterComparisonOperator $comparator): ?array
+    {
+        if (!is_int($value) && !is_numeric($value)) {
+            return null;
+        }
+
+        $size = max(0, (int) $value);
+
+        return match ($attribute) {
+            'min' => match ($comparator) {
+                FilterComparisonOperator::EQ, FilterComparisonOperator::GTE => ['size' => ['$gte' => $size]],
+                FilterComparisonOperator::GT => ['size' => ['$gt' => $size]],
+                FilterComparisonOperator::LT => ['size' => ['$lt' => $size]],
+                FilterComparisonOperator::LTE => ['size' => ['$lte' => $size]],
+                FilterComparisonOperator::NEQ => ['size' => ['$ne' => $size]],
+                default => null,
+            },
+            'max' => match ($comparator) {
+                FilterComparisonOperator::EQ, FilterComparisonOperator::LTE => ['size' => ['$lte' => $size]],
+                FilterComparisonOperator::LT => ['size' => ['$lt' => $size]],
+                FilterComparisonOperator::GT => ['size' => ['$gt' => $size]],
+                FilterComparisonOperator::GTE => ['size' => ['$gte' => $size]],
+                FilterComparisonOperator::NEQ => ['size' => ['$ne' => $size]],
+                default => null,
+            },
+            default => null,
+        };
+    }
+
+    /**
+     * Start of the (UTC) day a filter value falls on.
+     */
+    private function day(mixed $value): ?DateTimeImmutable
+    {
+        if ($value === null || $value === '') {
+            return null;
+        }
+
+        try {
+            $date = $value instanceof DateTimeImmutable ? $value : new DateTimeImmutable(trim((string) $value));
+        } catch (\Exception) {
+            return null;
+        }
+
+        return new DateTimeImmutable($date->format('Y-m-d'), new DateTimeZone('UTC'));
+    }
+}

lib/Stores/MessageStore.php

@@ -34,7 +34,7 @@ class MessageStore
     /** Fields dropped when a message becomes a tombstone */
     private const TOMBSTONE_UNSET = [
         'created' => '', 'received' => '', 'sent' => '', 'size' => '',
-        'subject' => '', 'from' => '', 'to' => '', 'cc' => '',
+        'subject' => '', 'from' => '', 'to' => '', 'cc' => '', 'bcc' => '',
         'urid' => '', 'inReplyTo' => '', 'references' => '',
         'flags' => '', 'hasAttachments' => '', 'preview' => '', 'blobs' => '',
     ];
@@ -179,6 +179,33 @@ class MessageStore
         return $uids;
     }
 
+    /**
+     * UIDs of the cached messages matching a filter, in sort order (tombstones excluded).
+     *
+     * @param array $filter meta store filter (see MessageQueryBuilder::filter())
+     * @param array<string, int> $sort meta store sort (see MessageQueryBuilder::sort())
+     * @param array|null $collation collation for text sorts
+     * @return int[]
+     */
+    public function query(string $serviceId, string $mailbox, int $uidValidity, array $filter = [], array $sort = ['uid' => 1], ?array $collation = null): array
+    {
+        $match = ['sid' => $serviceId, 'mailbox' => $mailbox, 'uidValidity' => $uidValidity, ...self::LIVE];
+        if ($filter !== []) {
+            $match = ['$and' => [$match, $filter]];
+        }
+
+        $options = ['projection' => ['uid' => 1], 'sort' => $sort];
+        if ($collation !== null) {
+            $options['collation'] = $collation;
+        }
+
+        $uids = [];
+        foreach ($this->dataStore->selectCollection(self::COLLECTION_NAME)->find($match, $options) as $document) {
+            $uids[] = (int) $document['uid'];
+        }
+        return $uids;
+    }
+
     /**
      * List the flags of every message cached for a mailbox.
      *

tests/php/Support/FakeLiveMailService.php

@@ -6,6 +6,7 @@ namespace KTXT\ProviderImap\Tests\Support;
 
 use Generator;
 use KTXF\Resource\Filter\IFilter;
+use KTXF\Resource\Range\IRange;
 use KTXF\Resource\Sort\ISort;
 use KTXM\ProviderImap\Client\Mailbox;
 use KTXM\ProviderImap\Client\Protocol\Command\Argument\FetchOptions;
@@ -75,6 +76,26 @@ final class FakeLiveMailService extends LiveMailService
         }
     }
 
+    /**
+     * Every message of the mailbox in UID order; filter / sort / range are not evaluated.
+     */
+    public function messageList(string $collection, ?IFilter $filter = null, ?ISort $sort = null, ?IRange $range = null): Generator
+    {
+        $this->calls[] = ['messageList', [$collection]];
+        $uids = array_keys($this->flags);
+        sort($uids);
+        yield from $this->messageFetch($collection, null, ...$uids);
+    }
+
+    /** @var int[] UIDs entityFind() returns */
+    public array $found = [];
+
+    public function entityFind(string $collection, ?IFilter $filter = null, ?ISort $sort = null, ?IRange $range = null): array
+    {
+        $this->calls[] = ['entityFind', [$collection, $filter]];
+        return $this->found;
+    }
+
     public function entityPatch(string $collection, array $flagsToAdd = [], array $flagsToRemove = [], int ...$uids): void
     {
         $this->calls[] = ['entityPatch', [$collection, $flagsToAdd, $flagsToRemove, $uids]];

tests/php/Support/FakeMessageFileStore.php

@@ -19,6 +19,11 @@ final class FakeMessageFileStore extends MessageFileStore
         $this->files[$uidValidity][$uid] = $content;
     }
 
+    public function read(string $tenantId, string $serviceId, string $mailbox, int $uidValidity, int $uid): ?array
+    {
+        return $this->files[$uidValidity][$uid] ?? null;
+    }
+
     public function delete(string $tenantId, string $serviceId, string $mailbox, int $uidValidity, int ...$uids): void
     {
         foreach ($uids as $uid) {

tests/php/Support/FakeMessageStore.php

@@ -20,6 +20,7 @@ final class FakeMessageStore extends MessageStore
         $meta = $entity->toCacheMeta();
         $existing = $this->documents[$uidValidity][$meta['uid']] ?? null;
         $this->documents[$uidValidity][$meta['uid']] = [
+            'meta' => $meta,
             'flags' => $meta['flags'],
             'addedSeq' => $existing['addedSeq'] ?? $seq,
             'modSeq' => $seq,
@@ -28,6 +29,30 @@ final class FakeMessageStore extends MessageStore
         ksort($this->documents[$uidValidity]);
     }
 
+    /** Last query() arguments, for assertions */
+    public ?array $lastQuery = null;
+
+    /**
+     * Live UIDs in ascending order; the filter / sort translation is covered by MessageQueryBuilderTest.
+     */
+    public function query(string $serviceId, string $mailbox, int $uidValidity, array $filter = [], array $sort = ['uid' => 1], ?array $collation = null): array
+    {
+        $this->lastQuery = [$filter, $sort, $collation];
+        return array_keys($this->flags($serviceId, $mailbox, $uidValidity));
+    }
+
+    public function fetchMany(string $serviceId, string $mailbox, int $uidValidity, int ...$uids): array
+    {
+        $list = [];
+        foreach ($uids as $uid) {
+            $document = $this->documents[$uidValidity][$uid] ?? null;
+            if ($document !== null && $document['removedSeq'] === null) {
+                $list[$uid] = [...$document['meta'], 'uid' => $uid, 'flags' => $document['flags']];
+            }
+        }
+        return $list;
+    }
+
     public function flags(string $serviceId, string $mailbox, int $uidValidity): array
     {
         $flags = [];

tests/php/Unit/CachedServiceTest.php

@@ -12,8 +12,12 @@ use KTXM\ProviderImap\Providers\MessageProperties;
 use KTXM\ProviderImap\Service\Cache\HarmonizationService;
 use KTXM\ProviderImap\Service\Cache\MessageDeltaService;
 use KTXM\ProviderImap\Service\Cache\MessageIngestor;
+use KTXM\ProviderImap\Service\Cache\MessageQueryBuilder;
 use KTXM\ProviderImap\Service\Live\LiveMailService;
 use KTXF\Resource\Identifier\EntityIdentifier;
+use KTXF\Resource\Range\RangeAnchorType;
+use KTXF\Resource\Range\RangeType;
+use KTXF\Resource\Sort\Sort;
 use KTXT\ProviderImap\Tests\Support\FakeLiveMailService;
 use KTXT\ProviderImap\Tests\Support\FakeMailboxStore;
 use KTXT\ProviderImap\Tests\Support\FakeMessageFileStore;
@@ -23,6 +27,8 @@ use PHPUnit\Framework\TestCase;
 final class CachedServiceTest extends TestCase
 {
     private FakeMailboxStore $mailboxes;
+    private FakeMessageStore $messages;
+    private FakeMessageFileStore $files;
     private FakeLiveMailService $live;
     private CachedService $service;
 
@@ -40,8 +46,14 @@ final class CachedServiceTest extends TestCase
             new MessageIngestor($files, $messages, $this->mailboxes),
         );
 
+        $this->messages = $messages;
+        $this->files = $files;
         $this->service = new CachedServiceTestDouble(
             $this->mailboxes,
+            $messages,
+            $files,
+            new MessageIngestor($files, $messages, $this->mailboxes),
+            new MessageQueryBuilder(),
             $harmonizer,
             new MessageDeltaService($this->mailboxes, $messages),
         );
@@ -152,6 +164,128 @@ final class CachedServiceTest extends TestCase
         $this->assertSame([5], $uids);
     }
 
+    public function testListOfANotHarmonizedMailboxStreamsFromTheServerAndCaches(): void
+    {
+        $this->live->flags = [1 => [], 2 => ['\\Seen']];
+
+        $uids = $this->listedUids($this->service->entityListStream('INBOX'));
+
+        $this->assertSame([1, 2], $uids);
+        $this->assertCount(1, $this->calls('messageList'));
+        $this->assertSame([1, 2], $this->files->uids(7));
+        $this->assertSame([1 => [], 2 => ['seen']], $this->messages->flags('svc', 'INBOX', 7));
+    }
+
+    public function testListOfAHarmonizedMailboxComesFromTheCache(): void
+    {
+        $this->harmonized([1 => [], 2 => ['\\Seen'], 3 => []]);
+
+        $entities = iterator_to_array($this->service->entityListStream('INBOX'), false);
+
+        $this->assertSame([1, 2, 3], array_map(static fn ($entity): int => (int) $entity->identifier(), $entities));
+        $this->assertSame([], $this->calls('messageList'));
+        $this->assertSame(['seen' => true], $entities[1]->getProperties()->getFlags());
+        $this->assertSame('INBOX', $entities[1]->collection());
+    }
+
+    public function testListRangeIsAppliedLikeTheLiveService(): void
+    {
+        $this->harmonized([1 => [], 2 => [], 3 => [], 4 => []]);
+
+        $absolute = $this->service->entityListRange(RangeType::TALLY);
+        $absolute->setAnchor(RangeAnchorType::ABSOLUTE);
+        $absolute->setPosition(1);
+        $absolute->setTally(2);
+
+        $relative = $this->service->entityListRange(RangeType::TALLY);
+        $relative->setAnchor(RangeAnchorType::RELATIVE);
+        $relative->setPosition(3);
+        $relative->setTally(2);
+
+        $this->assertSame([2, 3], $this->listedUids($this->service->entityListStream('INBOX', null, null, $absolute)));
+        $this->assertSame([3, 4], $this->listedUids($this->service->entityListStream('INBOX', null, null, $relative)));
+    }
+
+    public function testMessagesMissingFromTheContentStoreAreFetchedAndCached(): void
+    {
+        $this->harmonized([1 => [], 2 => []]);
+        $this->files->delete('tenant', 'svc', 'INBOX', 7, 2);
+        $this->live->fetchBatches = [];
+
+        $this->assertSame([1, 2], $this->listedUids($this->service->entityListStream('INBOX')));
+        $this->assertSame([[2]], $this->live->fetchBatches);
+        $this->assertSame([1, 2], $this->files->uids(7));
+    }
+
+    public function testBodyFilterHasTheServerFindTheUids(): void
+    {
+        $this->harmonized([1 => [], 2 => [], 3 => []]);
+        $this->live->found = [3, 1];
+        $this->live->fetchBatches = [];
+
+        $filter = $this->service->entityListFilter();
+        $filter->condition('body', 'invoice');
+
+        $this->assertSame([3, 1], $this->listedUids($this->service->entityListStream('INBOX', $filter)));
+        $this->assertCount(1, $this->calls('entityFind'));
+        $this->assertSame([], $this->live->fetchBatches, 'content comes from the cache');
+    }
+
+    public function testOtherFiltersAreTranslatedForTheMetaStore(): void
+    {
+        $this->harmonized([1 => []]);
+
+        $filter = $this->service->entityListFilter();
+        $filter->condition('subject', 'report');
+        // keyed attributes: entityListSort() passes a plain list, which Sort::condition() ignores (existing bug)
+        $sort = new Sort(['received' => true]);
+        $sort->condition('received', false);
+
+        iterator_to_array($this->service->entityListStream('INBOX', $filter, $sort));
+
+        $this->assertSame([], $this->calls('entityFind'));
+        $this->assertArrayHasKey('subject', $this->messages->lastQuery[0]);
+        $this->assertSame(['received' => -1, 'uid' => 1], $this->messages->lastQuery[1]);
+    }
+
+    public function testListBulkIsTheStreamCollected(): void
+    {
+        $this->harmonized([1 => [], 2 => []]);
+
+        $this->assertCount(2, $this->service->entityListBulk('INBOX'));
+    }
+
+    /**
+     * Harmonize INBOX with the given server UIDs and flags.
+     *
+     * @param array<int, list<string>> $flags
+     */
+    private function harmonized(array $flags): void
+    {
+        $this->live->flags = $flags;
+        $this->service->entityDelta('INBOX', '');
+    }
+
+    /**
+     * @return int[]
+     */
+    private function listedUids(iterable $entities): array
+    {
+        $uids = [];
+        foreach ($entities as $entity) {
+            $uids[] = (int) $entity->identifier();
+        }
+        return $uids;
+    }
+
+    /**
+     * @return list<array>
+     */
+    private function calls(string $method): array
+    {
+        return array_values(array_filter($this->live->calls, static fn (array $call): bool => $call[0] === $method));
+    }
+
     /**
      * @return list<array>
      */

tests/php/Unit/MessageQueryBuilderTest.php

@@ -0,0 +1,166 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KTXT\ProviderImap\Tests\Unit;
+
+use ArrayIterator;
+use KTXC\Db\Collection;
+use KTXC\Db\Cursor;
+use KTXC\Db\DataStore;
+use KTXF\Resource\Filter\FilterComparisonOperator;
+use KTXF\Resource\Filter\FilterConjunctionOperator;
+use KTXF\Resource\Filter\IFilter;
+use KTXF\Resource\Sort\Sort;
+use KTXM\ProviderImap\Service\Cache\MessageQueryBuilder;
+use KTXM\ProviderImap\Stores\MessageStore;
+use MongoDB\BSON\Regex;
+use PHPUnit\Framework\TestCase;
+
+final class MessageQueryBuilderTest extends TestCase
+{
+    private MessageQueryBuilder $builder;
+
+    protected function setUp(): void
+    {
+        $this->builder = new MessageQueryBuilder();
+    }
+
+    public function testNoFilterMatchesEverything(): void
+    {
+        $this->assertSame([], $this->builder->filter(null));
+        $this->assertFalse($this->builder->needsServer(null));
+    }
+
+    public function testBodyAndFullTextNeedTheServer(): void
+    {
+        $this->assertTrue($this->builder->needsServer($this->filter(['body', 'invoice'])));
+        $this->assertTrue($this->builder->needsServer($this->filter(['*', 'invoice'])));
+        $this->assertFalse($this->builder->needsServer($this->filter(['subject', 'invoice'])));
+    }
+
+    public function testSubjectIsACaseInsensitiveSubstring(): void
+    {
+        $filter = $this->builder->filter($this->filter(['subject', 'Q3 (draft)']));
+
+        $this->assertEquals(['subject' => new Regex('Q3 \\(draft\\)', 'i')], $filter);
+    }
+
+    public function testAddressesMatchAddressOrLabel(): void
+    {
+        $filter = $this->builder->filter($this->filter(['from', 'alice']));
+
+        $this->assertEquals(['$or' => [
+            ['from.address' => new Regex('alice', 'i')],
+            ['from.label' => new Regex('alice', 'i')],
+        ]], $filter);
+    }
+
+    public function testNegationAndMultipleValues(): void
+    {
+        $not = $this->builder->filter($this->filter(['subject', 'spam', FilterComparisonOperator::NLIKE]));
+        $any = $this->builder->filter($this->filter(['subject', ['a', 'b'], FilterComparisonOperator::IN]));
+
+        $this->assertEquals(['$nor' => [['subject' => new Regex('spam', 'i')]]], $not);
+        $this->assertEquals(['$or' => [['subject' => new Regex('a', 'i')], ['subject' => new Regex('b', 'i')]]], $any);
+    }
+
+    public function testConditionsCombineLeftToRightWithTheirConjunction(): void
+    {
+        $filter = $this->builder->filter($this->filter(
+            ['subject', 'a'],
+            ['subject', 'b', null, FilterConjunctionOperator::OR],
+            ['min', 100, FilterComparisonOperator::GTE, FilterConjunctionOperator::AND],
+        ));
+
+        $this->assertEquals(['$and' => [
+            ['$or' => [['subject' => new Regex('a', 'i')], ['subject' => new Regex('b', 'i')]]],
+            ['size' => ['$gte' => 100]],
+        ]], $filter);
+    }
+
+    public function testDatesCompareWholeUtcDaysOnReceived(): void
+    {
+        $before = $this->builder->filter($this->filter(['before', '2026-10-05T18:00:00+02:00', FilterComparisonOperator::LT]));
+        $after = $this->builder->filter($this->filter(['after', '2026-10-05', FilterComparisonOperator::GTE]));
+        $on = $this->builder->filter($this->filter(['after', '2026-10-05', FilterComparisonOperator::EQ]));
+
+        $this->assertSame(['received' => ['$lt' => '2026-10-05T00:00:00Z']], $before);
+        $this->assertSame(['received' => ['$gte' => '2026-10-05T00:00:00Z']], $after);
+        $this->assertSame(['received' => ['$gte' => '2026-10-05T00:00:00Z', '$lt' => '2026-10-06T00:00:00Z']], $on);
+    }
+
+    public function testSizes(): void
+    {
+        $this->assertSame(['size' => ['$lte' => 500]], $this->builder->filter($this->filter(['max', 500, FilterComparisonOperator::LTE])));
+        $this->assertSame(['size' => ['$gt' => 10]], $this->builder->filter($this->filter(['min', 10, FilterComparisonOperator::GT])));
+    }
+
+    public function testUnknownAttributesAndEmptyValuesAreIgnored(): void
+    {
+        $this->assertSame([], $this->builder->filter($this->filter(['unknown', 'x'], ['subject', '  '])));
+    }
+
+    public function testSortMapsFieldsAndEndsWithUid(): void
+    {
+        $sort = new Sort(['received' => true, 'subject' => true, 'to' => true]);
+        $sort->condition('received', false);
+
+        $this->assertSame(['sort' => ['received' => -1, 'uid' => 1], 'collation' => null], $this->builder->sort($sort));
+
+        $sort->condition('to', true);
+        $result = $this->builder->sort($sort);
+        $this->assertSame(['received' => -1, 'to.0.address' => 1, 'uid' => 1], $result['sort']);
+        $this->assertSame(['locale' => 'en', 'strength' => 2], $result['collation']);
+    }
+
+    public function testNoSortIsUidOrder(): void
+    {
+        $this->assertSame(['sort' => ['uid' => 1], 'collation' => null], $this->builder->sort(null));
+    }
+
+    public function testMetaStoreQueryExcludesTombstonesAndAppliesSortAndCollation(): void
+    {
+        $collection = $this->createMock(Collection::class);
+        $collection->expects($this->once())
+            ->method('find')
+            ->with(
+                ['$and' => [
+                    ['sid' => 'svc', 'mailbox' => 'INBOX', 'uidValidity' => 7, 'removedSeq' => null],
+                    ['size' => ['$gt' => 1]],
+                ]],
+                ['projection' => ['uid' => 1], 'sort' => ['subject' => 1, 'uid' => 1], 'collation' => ['locale' => 'en', 'strength' => 2]],
+            )
+            ->willReturn(new Cursor(new ArrayIterator([['uid' => 4], ['uid' => 2]])));
+        $dataStore = $this->createStub(DataStore::class);
+        $dataStore->method('selectCollection')->willReturn($collection);
+
+        $uids = (new MessageStore($dataStore))->query('svc', 'INBOX', 7, ['size' => ['$gt' => 1]], ['subject' => 1, 'uid' => 1], ['locale' => 'en', 'strength' => 2]);
+
+        $this->assertSame([4, 2], $uids);
+    }
+
+    /**
+     * A filter holding the given conditions as-is (Filter itself keeps one condition per attribute).
+     *
+     * @param array{0: string, 1: mixed, 2?: ?FilterComparisonOperator, 3?: ?FilterConjunctionOperator} ...$conditions
+     */
+    private function filter(array ...$conditions): IFilter
+    {
+        $list = array_map(static fn (array $condition): array => [
+            'attribute' => $condition[0],
+            'value' => $condition[1],
+            'comparator' => $condition[2] ?? FilterComparisonOperator::EQ,
+            'conjunction' => $condition[3] ?? null,
+        ], $conditions);
+
+        return new class($list) implements IFilter {
+            public function __construct(private readonly array $list) {}
+            public function attributes(): array { return []; }
+            public function comparators(): string { return ''; }
+            public function conjunctions(): string { return ''; }
+            public function condition(string $property, mixed $value, ?FilterComparisonOperator $comparator = null, ?FilterConjunctionOperator $conjunction = null): void {}
+            public function conditions(): array { return $this->list; }
+        };
+    }
+}