feat: serve message lists from the cache

Signed-off-by: Sebastian Krupinski <krupinski01@gmail.com>

lib/Providers/CachedService.php

@@ -22,11 +22,18 @@ use KTXF\Resource\Identifier\CollectionIdentifier;
 use KTXF\Resource\Identifier\EntityIdentifier;
 use KTXF\Resource\Identifier\EntityIdentifierInterface;
 use KTXF\Resource\Range\IRange;
+use KTXF\Resource\Range\IRangeTally;
+use KTXF\Resource\Range\RangeAnchorType;
 use KTXF\Resource\Sort\ISort;
 use KTXM\ProviderImap\Service\Cache\HarmonizationService;
 use KTXM\ProviderImap\Service\Cache\MessageDeltaService;
+use KTXM\ProviderImap\Service\Cache\MessageIngestor;
+use KTXM\ProviderImap\Service\Cache\MessageQueryBuilder;
 use KTXM\ProviderImap\Service\Live\LiveMailService;
 use KTXM\ProviderImap\Stores\MailboxStore;
+use KTXM\ProviderImap\Stores\MessageFileStore;
+use KTXM\ProviderImap\Stores\MessageStore;
+use UnexpectedValueException;
 
 /**
  * IMAP mail service that serves reads from the cache.
@@ -41,11 +48,18 @@ class CachedService extends ServiceBase
     /** Seconds after which a mailbox (or the mailbox list) is harmonized again on access */
     public const FRESHNESS_WINDOW = 60;
 
+    /** Messages hydrated from the cache per meta store query */
+    private const HYDRATE_BATCH_SIZE = 100;
+
     private ?LiveMailService $liveMail = null;
     private ?LiveService $live = null;
 
     public function __construct(
         private readonly MailboxStore $mailboxStore,
+        private readonly MessageStore $messageStore,
+        private readonly MessageFileStore $fileStore,
+        private readonly MessageIngestor $ingestor,
+        private readonly MessageQueryBuilder $queries,
         private readonly HarmonizationService $harmonizer,
         private readonly MessageDeltaService $deltas,
     ) {}
@@ -119,14 +133,42 @@ class CachedService extends ServiceBase
 
     public function entityListBulk(string|int $collection, ?IFilter $filter = null, ?ISort $sort = null, ?IRange $range = null, ?array $properties = null): array
     {
-        // served from the cache in 5.2b
-        return $this->live()->entityListBulk($collection, $filter, $sort, $range, $properties);
+        return iterator_to_array($this->entityListStream($collection, $filter, $sort, $range, $properties), true);
     }
 
+    /**
+     * List messages of a mailbox.
+     *
+     * - mailbox not harmonized yet: streamed from the server, each message cached as it passes
+     * - harmonized: UIDs from the meta store (filter / sort translated, range applied like the
+     *   live service), content from message.json; a filter on body / full text has the server
+     *   find the UIDs (IMAP SEARCH) and only the content comes from the cache
+     */
     public function entityListStream(string|int $collection, ?IFilter $filter = null, ?ISort $sort = null, ?IRange $range = null, ?array $properties = null): Generator
     {
-        // served from the cache in 5.2b
-        return $this->live()->entityListStream($collection, $filter, $sort, $range, $properties);
+        $mailbox = (string) $collection;
+        $state = $this->mailboxStore->state($this->serviceId(), $mailbox);
+
+        if (!$state['harmonizationComplete'] || $state['uidValidity'] === null) {
+            yield from $this->listFromServer($mailbox, $filter, $sort, $range);
+            return;
+        }
+
+        $uidValidity = (int) $state['uidValidity'];
+
+        if ($this->queries->needsServer($filter)) {
+            $uids = $this->liveMail()->entityFind($mailbox, $filter, $sort, $range);
+        } else {
+            $order = $this->queries->sort($sort);
+            $uids = self::applyRange(
+                $this->messageStore->query($this->serviceId(), $mailbox, $uidValidity, $this->queries->filter($filter), $order['sort'], $order['collation']),
+                $range,
+            );
+        }
+
+        foreach ($this->hydrate($mailbox, $uidValidity, $uids) as $entity) {
+            yield $entity->urn() => $entity;
+        }
     }
 
     public function entityFetchBulk(EntityIdentifierInterface ...$identifiers): array
@@ -211,6 +253,142 @@ class CachedService extends ServiceBase
 
     // ── Internals ────────────────────────────────────────────────────────────
 
+    /**
+     * Stream a list from the server, caching each message as it passes (cold mailbox).
+     */
+    private function listFromServer(string $mailbox, ?IFilter $filter, ?ISort $sort, ?IRange $range): Generator
+    {
+        $uidValidity = $this->cacheGeneration($mailbox);
+
+        foreach ($this->liveMail()->entityList($mailbox, $filter, $sort, $range) as $entity) {
+            if ($uidValidity !== null) {
+                $this->ingestQuietly($uidValidity, $entity);
+            }
+            yield $entity->urn() => $entity;
+        }
+    }
+
+    /**
+     * Entities for UIDs in the given order: cached content + meta flags, with messages
+     * missing from the cache (or written with another schema version) fetched from the
+     * server and cached.
+     *
+     * @param int[] $uids
+     * @return Generator<int, EntityResource>
+     */
+    private function hydrate(string $mailbox, int $uidValidity, array $uids): Generator
+    {
+        foreach (array_chunk($uids, self::HYDRATE_BATCH_SIZE) as $batch) {
+            $metas = $this->messageStore->fetchMany($this->serviceId(), $mailbox, $uidValidity, ...$batch);
+
+            $entities = [];
+            $missing = [];
+            foreach ($batch as $uid) {
+                $entity = isset($metas[$uid]) ? $this->entityFromCache($mailbox, $uidValidity, $metas[$uid]) : null;
+                if ($entity === null) {
+                    $missing[] = $uid;
+                    continue;
+                }
+                $entities[$uid] = $entity;
+            }
+
+            if ($missing !== []) {
+                foreach ($this->liveMail()->entityFetch($mailbox, ...$missing) as $uid => $entity) {
+                    $this->ingestQuietly($uidValidity, $entity);
+                    $entities[$uid] = $entity;
+                }
+            }
+
+            foreach ($batch as $uid) {
+                if (isset($entities[$uid])) {
+                    yield $uid => $entities[$uid];
+                }
+            }
+        }
+    }
+
+    /**
+     * An entity from its meta document and message.json; null when the content is missing or outdated.
+     */
+    private function entityFromCache(string $mailbox, int $uidValidity, array $meta): ?EntityResource
+    {
+        $content = $this->fileStore->read($this->tenantId(), $this->serviceId(), $mailbox, $uidValidity, (int) $meta['uid']);
+        if ($content === null) {
+            return null;
+        }
+
+        try {
+            return $this->entityFresh()->fromCacheContent($content)->fromCacheMeta($meta);
+        } catch (UnexpectedValueException) {
+            return null;
+        }
+    }
+
+    /**
+     * UIDVALIDITY to cache a not yet harmonized mailbox under; null when it cannot be cached.
+     */
+    private function cacheGeneration(string $mailbox): ?int
+    {
+        // the mailbox document holds the change sequence, so it has to exist before ingesting
+        if ($this->mailboxStore->fetch($this->serviceId(), $mailbox) === null) {
+            try {
+                $this->harmonizer()->harmonizeMailboxes();
+            } catch (\Throwable) {
+                return null;
+            }
+            if ($this->mailboxStore->fetch($this->serviceId(), $mailbox) === null) {
+                return null;
+            }
+        }
+
+        $state = $this->mailboxStore->state($this->serviceId(), $mailbox);
+        if ($state['uidValidity'] !== null) {
+            return (int) $state['uidValidity'];
+        }
+
+        return $this->liveMail()->mailboxFetch($mailbox)?->uidValidity();
+    }
+
+    /**
+     * Cache a message; a failing cache write must not fail the read that triggered it.
+     */
+    private function ingestQuietly(int $uidValidity, EntityResource $entity): void
+    {
+        try {
+            $this->ingestor->ingest($this->tenantId(), $this->serviceId(), $uidValidity, $entity);
+        } catch (\Throwable) {
+            // the next harmonization caches it
+        }
+    }
+
+    /**
+     * Apply a list range to sorted UIDs, as the live service does: absolute skips `position`
+     * messages, relative starts at the UID given as `position`.
+     *
+     * @param int[] $uids
+     * @return int[]
+     */
+    private static function applyRange(array $uids, ?IRange $range): array
+    {
+        if (!$range instanceof IRangeTally) {
+            return array_values($uids);
+        }
+
+        $tally = max(0, $range->getTally());
+        if ($tally === 0) {
+            return [];
+        }
+
+        if ($range->getAnchor() === RangeAnchorType::ABSOLUTE) {
+            $start = max(0, (int) $range->getPosition());
+        } else {
+            $index = array_search((int) $range->getPosition(), $uids, true);
+            $start = $index === false ? 0 : $index;
+        }
+
+        return array_values(array_slice($uids, $start, $tally));
+    }
+
     /**
      * Build a collection from its cached document; its signature is the delta signature of the mailbox.
      */
@@ -240,6 +418,11 @@ class CachedService extends ServiceBase
         return (string) $this->identifier();
     }
 
+    private function tenantId(): string
+    {
+        return (string) $this->tenantIdentifier();
+    }
+
     /**
      * One IMAP connection per service object, shared by the LiveService and harmonization.
      */

lib/Providers/MessageProperties.php

@@ -155,6 +155,7 @@ class MessageProperties extends MessagePropertiesMutableAbstract {
             'from'           => $this->data[static::PROPERTY_FROM] ?? null,
             'to'             => $this->data[static::PROPERTY_TO] ?? [],
             'cc'             => $this->data[static::PROPERTY_CC] ?? [],
+            'bcc'            => $this->data[static::PROPERTY_BCC] ?? [],
             'urid'           => $this->data[static::PROPERTY_URID] ?? null,
             'inReplyTo'      => $this->data[static::PROPERTY_IN_REPLY_TO] ?? null,
             'references'     => $this->data[static::PROPERTY_REFERENCES] ?? [],

lib/Service/Cache/MessageQueryBuilder.php

@@ -0,0 +1,233 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: Sebastian Krupinski <krupinski01@gmail.com>
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace KTXM\ProviderImap\Service\Cache;
+
+use DateTimeImmutable;
+use DateTimeZone;
+use KTXF\Resource\Filter\FilterComparisonOperator;
+use KTXF\Resource\Filter\FilterConjunctionOperator;
+use KTXF\Resource\Filter\IFilter;
+use KTXF\Resource\Sort\ISort;
+use MongoDB\BSON\Regex;
+
+/**
+ * Translates entity list filters and sorts into meta store queries.
+ *
+ * Mirrors LiveMailService's IMAP SEARCH / SORT translation so a cached list matches a
+ * live one: conditions are combined left to right with their conjunction, strings
+ * match as case-insensitive substrings, dates compare whole (UTC) days on `received`,
+ * sizes compare `size`. Conditions the cache cannot answer (body text, full text)
+ * make the whole filter a server filter.
+ */
+class MessageQueryBuilder
+{
+    /** Attributes that need the message body, which the meta store does not hold */
+    private const SERVER_ATTRIBUTES = ['body', '*', 'all'];
+
+    /** Address fields: matched on address and display label */
+    private const ADDRESS_ATTRIBUTES = ['from', 'to', 'cc', 'bcc'];
+
+    /** Sorts on text fields use a case-insensitive collation */
+    private const TEXT_SORTS = ['subject', 'from', 'to'];
+
+    /**
+     * Whether the filter has to be evaluated by the server (IMAP SEARCH).
+     */
+    public function needsServer(?IFilter $filter): bool
+    {
+        foreach ($filter?->conditions() ?? [] as $condition) {
+            if (in_array($condition['attribute'] ?? '', self::SERVER_ATTRIBUTES, true)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Meta store filter for a list filter; [] matches everything.
+     *
+     * Only valid when needsServer() is false.
+     */
+    public function filter(?IFilter $filter): array
+    {
+        $expression = null;
+
+        foreach ($filter?->conditions() ?? [] as $condition) {
+            $operand = $this->operand($condition);
+            if ($operand === null) {
+                continue;
+            }
+
+            if ($expression === null) {
+                $expression = $operand;
+                continue;
+            }
+
+            $expression = (($condition['conjunction'] ?? FilterConjunctionOperator::AND) === FilterConjunctionOperator::OR)
+                ? ['$or' => [$expression, $operand]]
+                : ['$and' => [$expression, $operand]];
+        }
+
+        return $expression ?? [];
+    }
+
+    /**
+     * Meta store sort for a list sort; always ends with uid so pages are stable.
+     *
+     * @return array{sort: array<string, int>, collation: ?array} collation is set when a text field is sorted
+     */
+    public function sort(?ISort $sort): array
+    {
+        $order = [];
+        $text = false;
+
+        foreach ($sort?->conditions() ?? [] as $condition) {
+            $attribute = $condition['attribute'] ?? '';
+            $field = match ($attribute) {
+                'from' => 'from.address',
+                'to' => 'to.0.address',
+                'subject' => 'subject',
+                'received' => 'received',
+                'sent' => 'sent',
+                'size' => 'size',
+                default => null,
+            };
+            if ($field === null) {
+                continue;
+            }
+
+            $order[$field] = ($condition['direction'] ?? true) ? 1 : -1;
+            $text = $text || in_array($attribute, self::TEXT_SORTS, true);
+        }
+
+        $order['uid'] ??= 1;
+
+        return ['sort' => $order, 'collation' => $text ? ['locale' => 'en', 'strength' => 2] : null];
+    }
+
+    /**
+     * @param array{attribute:string, value:mixed, comparator?:FilterComparisonOperator, conjunction?:FilterConjunctionOperator|null} $condition
+     */
+    private function operand(array $condition): ?array
+    {
+        $attribute = $condition['attribute'] ?? '';
+        $value = $condition['value'] ?? null;
+        $comparator = $condition['comparator'] ?? FilterComparisonOperator::EQ;
+
+        return match (true) {
+            $attribute === 'subject', in_array($attribute, self::ADDRESS_ATTRIBUTES, true) => $this->stringOperand($attribute, $value, $comparator),
+            $attribute === 'before', $attribute === 'after' => $this->dateOperand($attribute, $value, $comparator),
+            $attribute === 'min', $attribute === 'max' => $this->sizeOperand($attribute, $value, $comparator),
+            default => null,
+        };
+    }
+
+    private function stringOperand(string $attribute, mixed $value, FilterComparisonOperator $comparator): ?array
+    {
+        $values = is_array($value) ? array_values($value) : [$value];
+        $values = array_values(array_filter(array_map(
+            static fn (mixed $item): string => trim((string) $item),
+            $values,
+        ), static fn (string $item): bool => $item !== ''));
+
+        if ($values === []) {
+            return null;
+        }
+
+        $matches = array_map(fn (string $item): array => $this->stringMatch($attribute, $item), $values);
+
+        return match ($comparator) {
+            FilterComparisonOperator::EQ, FilterComparisonOperator::LIKE, FilterComparisonOperator::IN
+                => count($matches) === 1 ? $matches[0] : ['$or' => $matches],
+            FilterComparisonOperator::NEQ, FilterComparisonOperator::NLIKE, FilterComparisonOperator::NIN
+                => ['$nor' => $matches],
+            default => null,
+        };
+    }
+
+    private function stringMatch(string $attribute, string $value): array
+    {
+        $regex = new Regex(preg_quote($value, '/'), 'i');
+
+        if ($attribute === 'subject') {
+            return ['subject' => $regex];
+        }
+
+        return ['$or' => [[$attribute . '.address' => $regex], [$attribute . '.label' => $regex]]];
+    }
+
+    private function dateOperand(string $attribute, mixed $value, FilterComparisonOperator $comparator): ?array
+    {
+        $day = $this->day($value);
+        if ($day === null) {
+            return null;
+        }
+
+        $start = $day->format('Y-m-d\TH:i:s\Z');
+        $end = $day->modify('+1 day')->format('Y-m-d\TH:i:s\Z');
+        $on = ['received' => ['$gte' => $start, '$lt' => $end]];
+
+        return match ($comparator) {
+            FilterComparisonOperator::EQ => $on,
+            FilterComparisonOperator::NEQ => ['$nor' => [$on]],
+            FilterComparisonOperator::LT, FilterComparisonOperator::LTE => $attribute === 'before' ? ['received' => ['$lt' => $start]] : null,
+            FilterComparisonOperator::GT, FilterComparisonOperator::GTE => $attribute === 'after' ? ['received' => ['$gte' => $start]] : null,
+            default => null,
+        };
+    }
+
+    private function sizeOperand(string $attribute, mixed $value, FilterComparisonOperator $comparator): ?array
+    {
+        if (!is_int($value) && !is_numeric($value)) {
+            return null;
+        }
+
+        $size = max(0, (int) $value);
+
+        return match ($attribute) {
+            'min' => match ($comparator) {
+                FilterComparisonOperator::EQ, FilterComparisonOperator::GTE => ['size' => ['$gte' => $size]],
+                FilterComparisonOperator::GT => ['size' => ['$gt' => $size]],
+                FilterComparisonOperator::LT => ['size' => ['$lt' => $size]],
+                FilterComparisonOperator::LTE => ['size' => ['$lte' => $size]],
+                FilterComparisonOperator::NEQ => ['size' => ['$ne' => $size]],
+                default => null,
+            },
+            'max' => match ($comparator) {
+                FilterComparisonOperator::EQ, FilterComparisonOperator::LTE => ['size' => ['$lte' => $size]],
+                FilterComparisonOperator::LT => ['size' => ['$lt' => $size]],
+                FilterComparisonOperator::GT => ['size' => ['$gt' => $size]],
+                FilterComparisonOperator::GTE => ['size' => ['$gte' => $size]],
+                FilterComparisonOperator::NEQ => ['size' => ['$ne' => $size]],
+                default => null,
+            },
+            default => null,
+        };
+    }
+
+    /**
+     * Start of the (UTC) day a filter value falls on.
+     */
+    private function day(mixed $value): ?DateTimeImmutable
+    {
+        if ($value === null || $value === '') {
+            return null;
+        }
+
+        try {
+            $date = $value instanceof DateTimeImmutable ? $value : new DateTimeImmutable(trim((string) $value));
+        } catch (\Exception) {
+            return null;
+        }
+
+        return new DateTimeImmutable($date->format('Y-m-d'), new DateTimeZone('UTC'));
+    }
+}

lib/Stores/MessageStore.php

@@ -34,7 +34,7 @@ class MessageStore
     /** Fields dropped when a message becomes a tombstone */
     private const TOMBSTONE_UNSET = [
         'created' => '', 'received' => '', 'sent' => '', 'size' => '',
-        'subject' => '', 'from' => '', 'to' => '', 'cc' => '',
+        'subject' => '', 'from' => '', 'to' => '', 'cc' => '', 'bcc' => '',
         'urid' => '', 'inReplyTo' => '', 'references' => '',
         'flags' => '', 'hasAttachments' => '', 'preview' => '', 'blobs' => '',
     ];
@@ -179,6 +179,33 @@ class MessageStore
         return $uids;
     }
 
+    /**
+     * UIDs of the cached messages matching a filter, in sort order (tombstones excluded).
+     *
+     * @param array $filter meta store filter (see MessageQueryBuilder::filter())
+     * @param array<string, int> $sort meta store sort (see MessageQueryBuilder::sort())
+     * @param array|null $collation collation for text sorts
+     * @return int[]
+     */
+    public function query(string $serviceId, string $mailbox, int $uidValidity, array $filter = [], array $sort = ['uid' => 1], ?array $collation = null): array
+    {
+        $match = ['sid' => $serviceId, 'mailbox' => $mailbox, 'uidValidity' => $uidValidity, ...self::LIVE];
+        if ($filter !== []) {
+            $match = ['$and' => [$match, $filter]];
+        }
+
+        $options = ['projection' => ['uid' => 1], 'sort' => $sort];
+        if ($collation !== null) {
+            $options['collation'] = $collation;
+        }
+
+        $uids = [];
+        foreach ($this->dataStore->selectCollection(self::COLLECTION_NAME)->find($match, $options) as $document) {
+            $uids[] = (int) $document['uid'];
+        }
+        return $uids;
+    }
+
     /**
      * List the flags of every message cached for a mailbox.
      *