feat: implement meta and content cache

Signed-off-by: Sebastian Krupinski <krupinski01@gmail.com>

lib/Module.php

@@ -25,6 +25,8 @@ use KTXM\ProviderImap\Console\DisconnectCommand;
 use KTXM\ProviderImap\Console\TestCommand;
 use KTXM\ProviderImap\Listeners\UserEventListener;
 use KTXM\ProviderImap\Providers\Provider as MailProvider;
+use KTXM\ProviderImap\Stores\MailboxStore;
+use KTXM\ProviderImap\Stores\MessageStore;
 
 /**
  * IMAP Mail Provider Module
@@ -36,6 +38,8 @@ class Module extends ModuleInstanceAbstract
     public function __construct(
         private readonly ProviderManager $providerManager,
         private readonly EventListenerRegistrarInterface $events,
+        private readonly MailboxStore $mailboxStore,
+        private readonly MessageStore $messageStore,
     ) {}
 
     public function handle(): string
@@ -74,6 +78,16 @@ class Module extends ModuleInstanceAbstract
         ];
     }
 
+    public function install(): void
+    {
+        $this->ensureIndexes();
+    }
+
+    public function upgrade(): void
+    {
+        $this->ensureIndexes();
+    }
+
     public function boot(): void
     {
         // Register listeners
@@ -100,4 +114,13 @@ class Module extends ModuleInstanceAbstract
             }
         }
     }
+
+    /**
+     * Create the cache collection indexes (idempotent).
+     */
+    private function ensureIndexes(): void
+    {
+        $this->mailboxStore->ensureIndexes();
+        $this->messageStore->ensureIndexes();
+    }
 }

lib/Providers/CollectionProperties.php

@@ -16,8 +16,8 @@ use KTXF\Mail\Collection\CollectionRoles;
 /**
  * IMAP Mail Collection Properties
  *
- * Backed by the same internal $data shape as the JMAP provider so that cache
- * documents are interchangeable with fromStore() / toStore().
+ * Backed by the same internal $data shape as the JMAP provider so that
+ * cache documents are interchangeable with fromCacheMeta() / toCacheMeta().
  */
 class CollectionProperties extends CollectionPropertiesMutableAbstract
 {
@@ -50,14 +50,14 @@ class CollectionProperties extends CollectionPropertiesMutableAbstract
         return $this;
     }
 
-    // ── Store (MongoDB cache) ────────────────────────────────────────────────
+    // ── Cache (meta store) ───────────────────────────────────────────────────
 
-    public function toStore(): array
+    public function toCacheMeta(): array
     {
         return $this->data;
     }
 
-    public function fromStore(array $data): static
+    public function fromCacheMeta(array $data): static
     {
         $this->data = $data;
         return $this;

lib/Providers/CollectionResource.php

@@ -54,29 +54,38 @@ class CollectionResource extends CollectionMutableAbstract
         return $this;
     }
 
-    // ── Store (MongoDB cache) ────────────────────────────────────────────────
+    // ── Cache (meta store) ───────────────────────────────────────────────────
 
     /**
-     * Serialise to a MongoDB document.
+     * Serialise to a meta store document.
      *
-     * The caller must inject the service UUID as `sid` before persisting.
+     * The store adds the key fields it owns (sid, tid).
      */
-    public function toStore(): array
+    public function toCacheMeta(): array
     {
-        return array_merge(
-            $this->data,
-            [
-                'name'       => $this->data['identifier'],
-                'properties' => $this->getProperties()->toStore(),
-            ],
-        );
+        return [
+            'name'       => (string) $this->data[static::PROPERTY_IDENTIFIER],
+            'parent'     => $this->data[static::PROPERTY_COLLECTION] ?? null,
+            'signature'  => $this->data[static::PROPERTY_SIGNATURE] ?? null,
+            'properties' => $this->getProperties()->toCacheMeta(),
+        ];
     }
 
-    public function fromStore(array $data): static
+    /**
+     * Restore from a meta store document.
+     *
+     * Only the resource's own fields are read; store keys and harmonization state
+     * on the same document are ignored.
+     */
+    public function fromCacheMeta(array $data): static
     {
-        $this->data = $data;
+        $this->data[static::PROPERTY_IDENTIFIER] = (string) $data['name'];
+        $this->data[static::PROPERTY_COLLECTION] = $data['parent'] ?? null;
+        if (isset($data['signature'])) {
+            $this->data[static::PROPERTY_SIGNATURE] = $data['signature'];
+        }
         if (isset($data['properties'])) {
-            $this->getProperties()->fromStore($data['properties']);
+            $this->getProperties()->fromCacheMeta((array) $data['properties']);
         }
         return $this;
     }

lib/Providers/EntityResource.php

@@ -18,6 +18,9 @@ use KTXF\Mail\Object\MessagePropertiesMutableInterface;
  */
 class EntityResource extends EntityMutableAbstract {
 
+    /** Version of the meta store / content store formats; bump on any change to either */
+    public const CACHE_SCHEMA_VERSION = 1;
+
     public function __construct(
         string $provider = 'imap',
         string|int|null $service = null,
@@ -63,6 +66,69 @@ class EntityResource extends EntityMutableAbstract {
         return $this;
     }
 
+    // ── Cache (meta store / content store) ───────────────────────────────────
+
+    /**
+     * Serialise to a meta store document.
+     *
+     * The store adds the key fields it owns (sid, tid, uidValidity).
+     */
+    public function toCacheMeta(): array
+    {
+        return [
+            'mailbox'       => (string) $this->data['collection'],
+            'uid'           => (int) $this->data['identifier'],
+            'created'       => $this->data['created'] ?? null,
+            'schemaVersion' => self::CACHE_SCHEMA_VERSION,
+            ...$this->getProperties()->toCacheMeta(),
+        ];
+    }
+
+    /**
+     * Restore identity and mutable fields (flags) from a meta store document.
+     */
+    public function fromCacheMeta(array $document): static
+    {
+        $this->data['collection'] = (string) $document['mailbox'];
+        $this->data['identifier'] = (int) $document['uid'];
+
+        $this->getProperties()->fromCacheMeta($document);
+
+        return $this;
+    }
+
+    /**
+     * Serialise the immutable message content for the content store (message.json).
+     */
+    public function toCacheContent(): array
+    {
+        return [
+            'schemaVersion' => self::CACHE_SCHEMA_VERSION,
+            'created'       => $this->data['created'] ?? null,
+            'properties'    => $this->getProperties()->toCacheContent(),
+        ];
+    }
+
+    /**
+     * Restore the immutable message content from the content store (message.json).
+     *
+     * @throws \UnexpectedValueException when the content was written with a different schema version
+     */
+    public function fromCacheContent(array $content): static
+    {
+        if (($content['schemaVersion'] ?? null) !== self::CACHE_SCHEMA_VERSION) {
+            throw new \UnexpectedValueException('Cached message content has an unsupported schema version');
+        }
+
+        if (isset($content['created'])) {
+            $this->data['created'] = $content['created'];
+        }
+
+        $this->getProperties()->fromCacheContent($content['properties'] ?? []);
+
+        return $this;
+    }
+
     /**
      * @inheritDoc
      */

lib/Providers/MessageProperties.php

@@ -35,9 +35,9 @@ class MessageProperties extends MessagePropertiesMutableAbstract {
             $this->data[static::PROPERTY_IN_REPLY_TO] = $message->inReplyTo();
         }
 
-        //if ($message->references() !== []) {
-        //    $this->data[static::PROPERTY_REFERENCES] = $message->references();
-        //}
+        if ($message->references() !== []) {
+            $this->data[static::PROPERTY_REFERENCES] = $message->references();
+        }
 
         $receivedAt = $message->receivedAt() ?? $message->internalDate();
         if ($receivedAt !== null) {
@@ -109,6 +109,107 @@ class MessageProperties extends MessagePropertiesMutableAbstract {
         return $this;
     }
 
+    // ── Cache (meta store / content store) ───────────────────────────────────
+
+    /**
+     * Serialise the fields the meta store needs for list, filter and sort.
+     *
+     * Dates are normalised to UTC so they sort correctly as strings; the original
+     * values (with their offsets) are kept in the content store.
+     */
+    public function toCacheMeta(): array
+    {
+        return [
+            'received'       => self::cacheDate($this->data[static::PROPERTY_RECEIVED] ?? null),
+            'sent'           => self::cacheDate($this->data[static::PROPERTY_SENT] ?? null),
+            'size'           => $this->data[static::PROPERTY_SIZE] ?? 0,
+            'subject'        => $this->data[static::PROPERTY_SUBJECT] ?? '',
+            'from'           => $this->data[static::PROPERTY_FROM] ?? null,
+            'to'             => $this->data[static::PROPERTY_TO] ?? [],
+            'cc'             => $this->data[static::PROPERTY_CC] ?? [],
+            'urid'           => $this->data[static::PROPERTY_URID] ?? null,
+            'inReplyTo'      => $this->data[static::PROPERTY_IN_REPLY_TO] ?? null,
+            'references'     => $this->data[static::PROPERTY_REFERENCES] ?? [],
+            // list of set flags, not a map: keywords such as $Forwarded are not valid field names
+            'flags'          => array_keys(array_filter($this->data[static::PROPERTY_FLAGS] ?? [])),
+            'hasAttachments' => ($this->data[static::PROPERTY_ATTACHMENTS] ?? []) !== [],
+            'preview'        => $this->cachePreview(),
+        ];
+    }
+
+    /**
+     * Restore the mutable fields from a meta store document.
+     *
+     * Only flags are taken from the meta store; everything else comes from the
+     * content store, which holds the original values.
+     */
+    public function fromCacheMeta(array $document): static
+    {
+        $this->data[static::PROPERTY_FLAGS] = array_fill_keys(
+            array_map('strval', (array) ($document['flags'] ?? [])),
+            true,
+        );
+
+        return $this;
+    }
+
+    /**
+     * Serialise the immutable message content for the content store.
+     */
+    public function toCacheContent(): array
+    {
+        $content = $this->data;
+        unset($content[static::PROPERTY_FLAGS]);
+
+        return $content;
+    }
+
+    /**
+     * Restore the immutable message content from the content store, keeping any flags already set.
+     */
+    public function fromCacheContent(array $content): static
+    {
+        unset($content[static::PROPERTY_FLAGS]);
+        $flags = $this->data[static::PROPERTY_FLAGS] ?? null;
+
+        $this->data = $content;
+        if ($flags !== null) {
+            $this->data[static::PROPERTY_FLAGS] = $flags;
+        }
+
+        return $this;
+    }
+
+    private function cachePreview(int $length = 200): string
+    {
+        $text = $this->getBodyTextPlain();
+        if ($text === null) {
+            $html = $this->getBodyTextHtml();
+            $text = $html !== null
+                ? html_entity_decode(strip_tags((string) preg_replace('/<(style|script)\b[^>]*>.*?<\/\1>/is', ' ', $html)), ENT_QUOTES | ENT_HTML5, 'UTF-8')
+                : '';
+        }
+
+        $text = trim((string) preg_replace('/\s+/u', ' ', $text));
+
+        return mb_substr($text, 0, $length);
+    }
+
+    private static function cacheDate(?string $value): ?string
+    {
+        if ($value === null || $value === '') {
+            return null;
+        }
+
+        try {
+            return (new DateTimeImmutable($value))
+                ->setTimezone(new \DateTimeZone('UTC'))
+                ->format('Y-m-d\TH:i:s\Z');
+        } catch (\Exception) {
+            return null;
+        }
+    }
+
     /**
      * Recursively collect attachment parts from body structure
      */

lib/Stores/MailboxStore.php

@@ -0,0 +1,114 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: Sebastian Krupinski <krupinski01@gmail.com>
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace KTXM\ProviderImap\Stores;
+
+use KTXC\Db\DataStore;
+use KTXM\ProviderImap\Providers\CollectionResource;
+
+/**
+ * IMAP Mailbox Meta Store
+ *
+ * One MongoDB document per cached mailbox in `provider_imap_mail_mailboxes`,
+ * keyed by (sid, name). The document holds the mailbox itself and its
+ * harmonization state.
+ */
+class MailboxStore
+{
+    protected const COLLECTION_NAME = 'provider_imap_mail_mailboxes';
+
+    public function __construct(
+        protected readonly DataStore $dataStore,
+    ) {}
+
+    /**
+     * Create the collection indexes.
+     *
+     * MongoDB createIndex is idempotent when the name and specification match.
+     *
+     * @return string[]
+     */
+    public function ensureIndexes(): array
+    {
+        $collection = $this->dataStore->selectCollection(self::COLLECTION_NAME);
+
+        return [
+            $collection->createIndex(
+                ['sid' => 1, 'name' => 1],
+                ['name' => 'mailboxes_key', 'unique' => true]
+            ),
+            $collection->createIndex(
+                ['tid' => 1, 'sid' => 1],
+                ['name' => 'mailboxes_by_tenant_service']
+            ),
+        ];
+    }
+
+    /**
+     * Insert or update the mailbox fields of a document.
+     *
+     * Harmonization state on the same document is left untouched.
+     */
+    public function upsert(string $tenantId, string $serviceId, CollectionResource $collection): void
+    {
+        $document = $collection->toCacheMeta();
+
+        $this->dataStore->selectCollection(self::COLLECTION_NAME)->updateOne(
+            ['sid' => $serviceId, 'name' => $document['name']],
+            ['$set' => ['tid' => $tenantId, 'sid' => $serviceId, ...$document]],
+            ['upsert' => true],
+        );
+    }
+
+    /**
+     * Retrieve the document of a mailbox.
+     */
+    public function fetch(string $serviceId, string $name): ?array
+    {
+        return $this->dataStore->selectCollection(self::COLLECTION_NAME)->findOne([
+            'sid'  => $serviceId,
+            'name' => $name,
+        ]);
+    }
+
+    /**
+     * List the documents of all mailboxes of a service.
+     *
+     * @return array<string, array> keyed by mailbox name
+     */
+    public function list(string $serviceId): array
+    {
+        $cursor = $this->dataStore->selectCollection(self::COLLECTION_NAME)->find(['sid' => $serviceId]);
+
+        $list = [];
+        foreach ($cursor as $document) {
+            $list[(string) $document['name']] = $document;
+        }
+        return $list;
+    }
+
+    /**
+     * Delete the document of a mailbox.
+     */
+    public function delete(string $serviceId, string $name): void
+    {
+        $this->dataStore->selectCollection(self::COLLECTION_NAME)->deleteOne([
+            'sid'  => $serviceId,
+            'name' => $name,
+        ]);
+    }
+
+    /**
+     * Delete the documents of all mailboxes of a service.
+     */
+    public function deleteByService(string $serviceId): void
+    {
+        $this->dataStore->selectCollection(self::COLLECTION_NAME)->deleteMany(['sid' => $serviceId]);
+    }
+}

lib/Stores/MessageStore.php

@@ -0,0 +1,196 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: Sebastian Krupinski <krupinski01@gmail.com>
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace KTXM\ProviderImap\Stores;
+
+use KTXC\Db\DataStore;
+use KTXM\ProviderImap\Providers\EntityResource;
+
+/**
+ * IMAP Message Meta Store
+ *
+ * One MongoDB document per cached message in `provider_imap_mail_messages`,
+ * keyed by (sid, mailbox, uidValidity, uid). Holds only what list, filter and
+ * sort need; message content lives in the content store (message.json).
+ */
+class MessageStore
+{
+    protected const COLLECTION_NAME = 'provider_imap_mail_messages';
+
+    public function __construct(
+        protected readonly DataStore $dataStore,
+    ) {}
+
+    /**
+     * Create the collection indexes.
+     *
+     * MongoDB createIndex is idempotent when the name and specification match.
+     *
+     * @return string[]
+     */
+    public function ensureIndexes(): array
+    {
+        $collection = $this->dataStore->selectCollection(self::COLLECTION_NAME);
+
+        return [
+            $collection->createIndex(
+                ['sid' => 1, 'mailbox' => 1, 'uidValidity' => 1, 'uid' => 1],
+                ['name' => 'messages_key', 'unique' => true]
+            ),
+            $collection->createIndex(
+                ['sid' => 1, 'mailbox' => 1, 'received' => -1],
+                ['name' => 'messages_by_received']
+            ),
+            $collection->createIndex(
+                ['sid' => 1, 'mailbox' => 1, 'sent' => -1],
+                ['name' => 'messages_by_sent']
+            ),
+            $collection->createIndex(
+                ['sid' => 1, 'mailbox' => 1, 'flags' => 1],
+                ['name' => 'messages_by_flags']
+            ),
+            $collection->createIndex(
+                ['sid' => 1, 'urid' => 1],
+                ['name' => 'messages_by_urid']
+            ),
+            $collection->createIndex(
+                ['tid' => 1, 'sid' => 1],
+                ['name' => 'messages_by_tenant_service']
+            ),
+        ];
+    }
+
+    /**
+     * Insert or replace the meta document of a message.
+     *
+     * Fields owned by other writers (e.g. blobs) are left untouched.
+     */
+    public function upsert(string $tenantId, string $serviceId, int $uidValidity, EntityResource $entity): void
+    {
+        $document = $entity->toCacheMeta();
+        $key = [
+            'sid'         => $serviceId,
+            'mailbox'     => $document['mailbox'],
+            'uidValidity' => $uidValidity,
+            'uid'         => $document['uid'],
+        ];
+
+        $this->dataStore->selectCollection(self::COLLECTION_NAME)->updateOne(
+            $key,
+            ['$set' => ['tid' => $tenantId, ...$key, ...$document]],
+            ['upsert' => true],
+        );
+    }
+
+    /**
+     * Retrieve the meta document of a message.
+     */
+    public function fetch(string $serviceId, string $mailbox, int $uidValidity, int $uid): ?array
+    {
+        return $this->dataStore->selectCollection(self::COLLECTION_NAME)->findOne([
+            'sid'         => $serviceId,
+            'mailbox'     => $mailbox,
+            'uidValidity' => $uidValidity,
+            'uid'         => $uid,
+        ]);
+    }
+
+    /**
+     * Retrieve the meta documents of several messages.
+     *
+     * @return array<int, array> keyed by UID; UIDs that are not cached are absent
+     */
+    public function fetchMany(string $serviceId, string $mailbox, int $uidValidity, int ...$uids): array
+    {
+        if ($uids === []) {
+            return [];
+        }
+
+        $cursor = $this->dataStore->selectCollection(self::COLLECTION_NAME)->find([
+            'sid'         => $serviceId,
+            'mailbox'     => $mailbox,
+            'uidValidity' => $uidValidity,
+            'uid'         => ['$in' => array_values($uids)],
+        ]);
+
+        $list = [];
+        foreach ($cursor as $document) {
+            $list[(int) $document['uid']] = $document;
+        }
+        return $list;
+    }
+
+    /**
+     * List the UIDs cached for a mailbox.
+     *
+     * @return int[]
+     */
+    public function uids(string $serviceId, string $mailbox, int $uidValidity): array
+    {
+        $cursor = $this->dataStore->selectCollection(self::COLLECTION_NAME)->find(
+            ['sid' => $serviceId, 'mailbox' => $mailbox, 'uidValidity' => $uidValidity],
+            ['projection' => ['uid' => 1]],
+        );
+
+        $uids = [];
+        foreach ($cursor as $document) {
+            $uids[] = (int) $document['uid'];
+        }
+        return $uids;
+    }
+
+    /**
+     * Replace the flags of a message.
+     *
+     * @param string[] $flags set flags, e.g. ['seen', 'flagged']
+     */
+    public function updateFlags(string $serviceId, string $mailbox, int $uidValidity, int $uid, array $flags): void
+    {
+        $this->dataStore->selectCollection(self::COLLECTION_NAME)->updateOne(
+            ['sid' => $serviceId, 'mailbox' => $mailbox, 'uidValidity' => $uidValidity, 'uid' => $uid],
+            ['$set' => ['flags' => array_values($flags)]],
+        );
+    }
+
+    /**
+     * Delete the meta documents of messages.
+     */
+    public function delete(string $serviceId, string $mailbox, int $uidValidity, int ...$uids): void
+    {
+        if ($uids === []) {
+            return;
+        }
+
+        $this->dataStore->selectCollection(self::COLLECTION_NAME)->deleteMany([
+            'sid'         => $serviceId,
+            'mailbox'     => $mailbox,
+            'uidValidity' => $uidValidity,
+            'uid'         => ['$in' => array_values($uids)],
+        ]);
+    }
+
+    /**
+     * Delete all meta documents of a mailbox (any UIDVALIDITY).
+     */
+    public function deleteByMailbox(string $serviceId, string $mailbox): void
+    {
+        $this->dataStore->selectCollection(self::COLLECTION_NAME)->deleteMany([
+            'sid'     => $serviceId,
+            'mailbox' => $mailbox,
+        ]);
+    }
+
+    /**
+     * Delete all meta documents of a service.
+     */
+    public function deleteByService(string $serviceId): void
+    {
+        $this->dataStore->selectCollection(self::COLLECTION_NAME)->deleteMany(['sid' => $serviceId]);
+    }
+}