feat: track cache changes with sequence numbers and tombstones

Signed-off-by: Sebastian Krupinski <krupinski01@gmail.com>

lib/Service/Cache/HarmonizationService.php

@@ -184,7 +184,8 @@ class HarmonizationService
 
                 $flags = MessageProperties::normalizeFlags($flags);
                 if (!self::sameFlags($flags, $cachedFlags[$uid])) {
-                    $this->messageStore->updateFlags($serviceId, $mailbox, $uidValidity, $uid, $flags);
+                    $seq = $this->mailboxStore->reserveSequence($serviceId, $mailbox);
+                    $this->messageStore->updateFlags($serviceId, $mailbox, $uidValidity, $uid, $flags, $seq);
                     $result['updated']++;
                 }
                 unset($cachedFlags[$uid]);
@@ -199,10 +200,11 @@ class HarmonizationService
                 array_push($missing, ...array_diff($batch, $ingested));
             }
 
-            // expunged: cached UIDs the server did not list; meta documents first, then files
+            // expunged: cached UIDs the server did not list; tombstones first, then files
             $expunged = array_keys($cachedFlags);
             if ($expunged !== []) {
-                $this->messageStore->delete($serviceId, $mailbox, $uidValidity, ...$expunged);
+                $seq = $this->mailboxStore->reserveSequence($serviceId, $mailbox);
+                $this->messageStore->tombstone($serviceId, $mailbox, $uidValidity, $seq, ...$expunged);
                 $this->fileStore->delete($tenantId, $serviceId, $mailbox, $uidValidity, ...$expunged);
                 $result['removed'] = count($expunged);
             }

lib/Service/Cache/MessageDeltaService.php

@@ -0,0 +1,88 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: Sebastian Krupinski <krupinski01@gmail.com>
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace KTXM\ProviderImap\Service\Cache;
+
+use KTXF\Resource\Delta\Delta;
+use KTXF\Resource\Delta\DeltaCollection;
+use KTXM\ProviderImap\Stores\MailboxStore;
+use KTXM\ProviderImap\Stores\MessageStore;
+
+/**
+ * Answers "what changed since this signature?" for a cached mailbox.
+ *
+ * A signature is `<uidValidity>:<changeSeq>`. Read-only: harmonizing before
+ * answering is the caller's decision.
+ */
+class MessageDeltaService
+{
+    public function __construct(
+        private readonly MailboxStore $mailboxStore,
+        private readonly MessageStore $messageStore,
+    ) {}
+
+    /**
+     * Changes of a mailbox since a signature.
+     *
+     * - empty signature: the current signature and no changes, so the client gets a
+     *   starting point; while the initial harmonization is incomplete the signature is
+     *   `<uidValidity>:0`, so additions of the run in progress are not skipped
+     * - signature of another UIDVALIDITY, older than the purged tombstones, or not
+     *   understood: a reset, answered as changes since 0 (every message is an addition)
+     * - otherwise: additions, modifications and deletions after the signature
+     *
+     * A mailbox that is not cached (or never harmonized) yields an empty delta.
+     */
+    public function delta(string $serviceId, string $mailbox, string $signature): Delta
+    {
+        $state = $this->mailboxStore->state($serviceId, $mailbox);
+        if ($state['uidValidity'] === null) {
+            return new Delta();
+        }
+
+        $uidValidity = (int) $state['uidValidity'];
+        $current = self::signature($uidValidity, (int) $state['changeSeq']);
+
+        if ($signature === '') {
+            return new Delta(signature: $state['harmonizationComplete'] ? $current : self::signature($uidValidity, 0));
+        }
+
+        $since = self::since($signature, $uidValidity, (int) $state['purgedSeq'], (int) $state['changeSeq']) ?? 0;
+        $changes = $this->messageStore->changes($serviceId, $mailbox, $uidValidity, $since);
+
+        return new Delta(
+            new DeltaCollection(array_map('strval', $changes['additions'])),
+            new DeltaCollection(array_map('strval', $changes['modifications'])),
+            new DeltaCollection(array_map('strval', $changes['deletions'])),
+            $current,
+        );
+    }
+
+    public static function signature(int $uidValidity, int $changeSeq): string
+    {
+        return $uidValidity . ':' . $changeSeq;
+    }
+
+    /**
+     * The change sequence a signature refers to, or null when it needs a reset.
+     */
+    private static function since(string $signature, int $uidValidity, int $purgedSeq, int $changeSeq): ?int
+    {
+        if (preg_match('/^(\d+):(\d+)$/', $signature, $matches) !== 1) {
+            return null;
+        }
+
+        $since = (int) $matches[2];
+        if ((int) $matches[1] !== $uidValidity || $since < $purgedSeq || $since > $changeSeq) {
+            return null;
+        }
+
+        return $since;
+    }
+}

lib/Service/Cache/MessageIngestor.php

@@ -10,6 +10,7 @@ declare(strict_types=1);
 namespace KTXM\ProviderImap\Service\Cache;
 
 use KTXM\ProviderImap\Providers\EntityResource;
+use KTXM\ProviderImap\Stores\MailboxStore;
 use KTXM\ProviderImap\Stores\MessageFileStore;
 use KTXM\ProviderImap\Stores\MessageStore;
 
@@ -26,28 +27,37 @@ class MessageIngestor
     public function __construct(
         private readonly MessageFileStore $fileStore,
         private readonly MessageStore $messageStore,
+        private readonly MailboxStore $mailboxStore,
     ) {}
 
     /**
      * Cache messages of one mailbox generation.
      *
      * The content is written before the meta document, so an interruption leaves at
-     * most an orphaned file, never a meta document without content.
+     * most an orphaned file, never a meta document without content. Each message is
+     * stamped with its own change sequence, reserved per mailbox in one step.
      *
      * @return int[] UIDs that were cached
      */
     public function ingest(string $tenantId, string $serviceId, int $uidValidity, EntityResource ...$entities): array
     {
-        $ingested = [];
-
+        $byMailbox = [];
         foreach ($entities as $entity) {
-            $mailbox = (string) $entity->collection();
-            $uid = (int) $entity->identifier();
+            $byMailbox[(string) $entity->collection()][] = $entity;
+        }
 
-            $this->fileStore->write($tenantId, $serviceId, $mailbox, $uidValidity, $uid, $entity->toCacheContent());
-            $this->messageStore->upsert($tenantId, $serviceId, $uidValidity, $entity);
+        $ingested = [];
+        foreach ($byMailbox as $mailbox => $group) {
+            $seq = $this->mailboxStore->reserveSequence($serviceId, (string) $mailbox, count($group));
 
-            $ingested[] = $uid;
+            foreach ($group as $entity) {
+                $uid = (int) $entity->identifier();
+
+                $this->fileStore->write($tenantId, $serviceId, (string) $mailbox, $uidValidity, $uid, $entity->toCacheContent());
+                $this->messageStore->upsert($tenantId, $serviceId, $uidValidity, $entity, $seq++);
+
+                $ingested[] = $uid;
+            }
         }
 
         return $ingested;