feat: harmonize mailbox list and track harmonization state

Signed-off-by: Sebastian Krupinski <krupinski01@gmail.com>

lib/Providers/Service.php

@@ -237,6 +237,11 @@ class Service implements ServiceBaseInterface, ServiceMutableInterface, ServiceC
         return $this->serviceIdentifier;
     }
 
+    public function tenantIdentifier(): ?string
+    {
+        return $this->serviceTenantId;
+    }
+
     public function getLabel(): ?string
     {
         return $this->serviceLabel;

lib/Service/Cache/HarmonizationService.php

@@ -0,0 +1,125 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: Sebastian Krupinski <krupinski01@gmail.com>
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace KTXM\ProviderImap\Service\Cache;
+
+use KTXM\ProviderImap\Providers\CollectionResource;
+use KTXM\ProviderImap\Providers\Service;
+use KTXM\ProviderImap\Service\Live\LiveMailService;
+use KTXM\ProviderImap\Stores\MailboxStore;
+use KTXM\ProviderImap\Stores\MessageFileStore;
+use KTXM\ProviderImap\Stores\MessageStore;
+use LogicException;
+
+/**
+ * Brings the cache of one service in line with its IMAP server.
+ *
+ * Created by the container without a service; call for() to bind one. Only ever
+ * talks to LiveMailService, never to the cache it maintains.
+ */
+class HarmonizationService
+{
+    private ?Service $service = null;
+    private ?LiveMailService $live = null;
+
+    public function __construct(
+        private readonly MailboxStore $mailboxStore,
+        private readonly MessageStore $messageStore,
+        private readonly MessageFileStore $fileStore,
+    ) {}
+
+    /**
+     * Bind a service; returns a new instance so the shared one stays unbound.
+     */
+    public function for(Service $service, ?LiveMailService $live = null): static
+    {
+        $bound = clone $this;
+        $bound->service = $service;
+        $bound->live = $live ?? new LiveMailService($service);
+
+        return $bound;
+    }
+
+    /**
+     * Harmonize the mailbox list of the service with the server.
+     *
+     * New mailboxes are added (not yet harmonized), existing ones updated, and
+     * mailboxes gone from the server purged with their messages. A rename shows
+     * up as one mailbox gone and one new.
+     *
+     * @return array{added: string[], updated: string[], removed: string[]}
+     */
+    public function harmonizeMailboxes(): array
+    {
+        [$service, $live] = $this->bound();
+        $tenantId = (string) $service->tenantIdentifier();
+        $serviceId = (string) $service->identifier();
+
+        $remote = [];
+        foreach ($live->collectionList() as $name => $mailbox) {
+            $remote[(string) $name] = (new CollectionResource($service->provider(), $serviceId))->fromImap($mailbox);
+        }
+        $cached = $this->mailboxStore->list($serviceId);
+
+        $result = ['added' => [], 'updated' => [], 'removed' => []];
+
+        foreach ($remote as $name => $collection) {
+            $this->mailboxStore->upsert($tenantId, $serviceId, $collection);
+            $result[isset($cached[$name]) ? 'updated' : 'added'][] = $name;
+        }
+
+        // every IMAP account has an INBOX; a list without one is incomplete, so nothing is purged
+        if (!self::containsInbox(array_keys($remote))) {
+            return $result;
+        }
+
+        foreach (array_keys(array_diff_key($cached, $remote)) as $name) {
+            $this->purgeMailbox($tenantId, $serviceId, (string) $name);
+            $result['removed'][] = (string) $name;
+        }
+
+        return $result;
+    }
+
+    /**
+     * Remove a mailbox and everything cached for it: meta documents first, then files, then the mailbox.
+     */
+    private function purgeMailbox(string $tenantId, string $serviceId, string $name): void
+    {
+        $this->messageStore->deleteByMailbox($serviceId, $name);
+        $this->fileStore->deleteByMailbox($tenantId, $serviceId, $name);
+        $this->mailboxStore->delete($serviceId, $name);
+    }
+
+    /**
+     * @return array{0: Service, 1: LiveMailService}
+     */
+    private function bound(): array
+    {
+        if ($this->service === null || $this->live === null) {
+            throw new LogicException('HarmonizationService is not bound to a service; call for() first');
+        }
+
+        return [$this->service, $this->live];
+    }
+
+    /**
+     * @param string[] $names
+     */
+    private static function containsInbox(array $names): bool
+    {
+        foreach ($names as $name) {
+            if (strcasecmp($name, 'INBOX') === 0) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+}

lib/Stores/MailboxStore.php

@@ -23,6 +23,16 @@ class MailboxStore
 {
     protected const COLLECTION_NAME = 'provider_imap_mail_mailboxes';
 
+    /** Harmonization state fields and their defaults for a mailbox that was never harmonized */
+    public const STATE_DEFAULTS = [
+        'uidValidity'           => null,
+        'uidNext'               => null,
+        'highestModSeq'         => null,
+        'changeSeq'             => 0,
+        'harmonizedAt'          => null,
+        'harmonizationComplete' => false,
+    ];
+
     public function __construct(
         protected readonly DataStore $dataStore,
     ) {}
@@ -53,7 +63,8 @@ class MailboxStore
     /**
      * Insert or update the mailbox fields of a document.
      *
-     * Harmonization state on the same document is left untouched.
+     * Harmonization state on an existing document is left untouched; a new document
+     * starts with the default (never harmonized) state.
      */
     public function upsert(string $tenantId, string $serviceId, CollectionResource $collection): void
     {
@@ -61,7 +72,10 @@ class MailboxStore
 
         $this->dataStore->selectCollection(self::COLLECTION_NAME)->updateOne(
             ['sid' => $serviceId, 'name' => $document['name']],
-            ['$set' => ['tid' => $tenantId, 'sid' => $serviceId, ...$document]],
+            [
+                '$set' => ['tid' => $tenantId, 'sid' => $serviceId, ...$document],
+                '$setOnInsert' => self::STATE_DEFAULTS,
+            ],
             ['upsert' => true],
         );
     }
@@ -93,6 +107,75 @@ class MailboxStore
         return $list;
     }
 
+    // ── Harmonization state ──────────────────────────────────────────────────
+
+    /**
+     * Harmonization state of a mailbox; defaults when the mailbox is not cached.
+     *
+     * @return array{uidValidity: ?int, uidNext: ?int, highestModSeq: ?int, changeSeq: int, harmonizedAt: ?int, harmonizationComplete: bool}
+     */
+    public function state(string $serviceId, string $name): array
+    {
+        $document = $this->fetch($serviceId, $name) ?? [];
+
+        return array_replace(self::STATE_DEFAULTS, array_intersect_key($document, self::STATE_DEFAULTS));
+    }
+
+    /**
+     * Update harmonization state fields of a mailbox; keys outside the state are ignored.
+     */
+    public function updateState(string $serviceId, string $name, array $state): void
+    {
+        $state = array_intersect_key($state, self::STATE_DEFAULTS);
+        if ($state === []) {
+            return;
+        }
+
+        $this->dataStore->selectCollection(self::COLLECTION_NAME)->updateOne(
+            ['sid' => $serviceId, 'name' => $name],
+            ['$set' => $state],
+        );
+    }
+
+    /**
+     * Take the harmonization lock of a mailbox when it is free or expired.
+     *
+     * Never waits: returns false when another owner holds a valid lock or the
+     * mailbox is not cached.
+     */
+    public function acquireLock(string $serviceId, string $name, string $owner, int $ttl): bool
+    {
+        $now = time();
+
+        $result = $this->dataStore->selectCollection(self::COLLECTION_NAME)->updateOne(
+            [
+                'sid'  => $serviceId,
+                'name' => $name,
+                '$or'  => [
+                    ['lock' => null],
+                    ['lock.expiresAt' => ['$lte' => $now]],
+                    ['lock.owner' => $owner],
+                ],
+            ],
+            ['$set' => ['lock' => ['owner' => $owner, 'expiresAt' => $now + $ttl]]],
+        );
+
+        return $result->getMatchedCount() === 1;
+    }
+
+    /**
+     * Release the harmonization lock of a mailbox, if still held by the owner.
+     */
+    public function releaseLock(string $serviceId, string $name, string $owner): void
+    {
+        $this->dataStore->selectCollection(self::COLLECTION_NAME)->updateOne(
+            ['sid' => $serviceId, 'name' => $name, 'lock.owner' => $owner],
+            ['$unset' => ['lock' => '']],
+        );
+    }
+
+    // ── Deletion ─────────────────────────────────────────────────────────────
+
     /**
      * Delete the document of a mailbox.
      */

tests/php/Unit/HarmonizationServiceTest.php

@@ -0,0 +1,173 @@
+<?php
+
+declare(strict_types=1);
+
+namespace KTXT\ProviderImap\Tests\Unit;
+
+use Generator;
+use KTXC\Db\Collection;
+use KTXC\Db\DataStore;
+use KTXF\Resource\Filter\IFilter;
+use KTXF\Resource\Sort\ISort;
+use KTXM\ProviderImap\Client\Mailbox;
+use KTXM\ProviderImap\Providers\CollectionResource;
+use KTXM\ProviderImap\Providers\Service;
+use KTXM\ProviderImap\Service\Cache\HarmonizationService;
+use KTXM\ProviderImap\Service\Live\LiveMailService;
+use KTXM\ProviderImap\Stores\MailboxStore;
+use KTXM\ProviderImap\Stores\MessageFileStore;
+use KTXM\ProviderImap\Stores\MessageStore;
+use LogicException;
+use MongoDB\UpdateResult;
+use PHPUnit\Framework\TestCase;
+
+final class HarmonizationServiceTest extends TestCase
+{
+    private array $calls = [];
+
+    public function testMailboxesAreAddedUpdatedAndRemoved(): void
+    {
+        $service = $this->harmonizer(
+            cached: ['INBOX' => [], 'Old' => []],
+            remote: ['INBOX', 'Reports'],
+        );
+
+        $result = $service->harmonizeMailboxes();
+
+        $this->assertSame(['added' => ['Reports'], 'updated' => ['INBOX'], 'removed' => ['Old']], $result);
+        $this->assertSame([
+            ['upsert', 'tenant', 'svc', 'INBOX'],
+            ['upsert', 'tenant', 'svc', 'Reports'],
+            ['messages.deleteByMailbox', 'svc', 'Old'],
+            ['files.deleteByMailbox', 'tenant', 'svc', 'Old'],
+            ['mailboxes.delete', 'svc', 'Old'],
+        ], $this->calls);
+    }
+
+    public function testNothingIsPurgedWhenTheServerListHasNoInbox(): void
+    {
+        $service = $this->harmonizer(
+            cached: ['INBOX' => [], 'Reports' => []],
+            remote: [],
+        );
+
+        $result = $service->harmonizeMailboxes();
+
+        $this->assertSame(['added' => [], 'updated' => [], 'removed' => []], $result);
+        $this->assertSame([], $this->calls);
+    }
+
+    public function testUnboundServiceIsRejected(): void
+    {
+        $unbound = new HarmonizationService(
+            $this->createStub(MailboxStore::class),
+            $this->createStub(MessageStore::class),
+            $this->createStub(MessageFileStore::class),
+        );
+
+        $this->expectException(LogicException::class);
+        $unbound->harmonizeMailboxes();
+    }
+
+    public function testLockIsTakenOnlyWhenFreeExpiredOrOwned(): void
+    {
+        $collection = $this->createMock(Collection::class);
+        $collection->expects($this->exactly(2))
+            ->method('updateOne')
+            ->willReturnCallback(function (array $filter, array $update) use (&$matched): UpdateResult {
+                $this->assertSame(['lock' => null], $filter['$or'][0]);
+                $this->assertArrayHasKey('$lte', $filter['$or'][1]['lock.expiresAt']);
+                $this->assertSame(['lock.owner' => 'run-1'], $filter['$or'][2]);
+                $this->assertSame('run-1', $update['$set']['lock']['owner']);
+                $result = $this->createStub(UpdateResult::class);
+                $result->method('getMatchedCount')->willReturn(array_shift($matched));
+                return $result;
+            });
+        $matched = [1, 0];
+
+        $store = new MailboxStore($this->dataStore($collection));
+
+        $this->assertTrue($store->acquireLock('svc', 'INBOX', 'run-1', 300));
+        $this->assertFalse($store->acquireLock('svc', 'INBOX', 'run-1', 300));
+    }
+
+    public function testStateFillsDefaultsAndIgnoresOtherFields(): void
+    {
+        $collection = $this->createStub(Collection::class);
+        $collection->method('findOne')->willReturn([
+            'sid' => 'svc', 'name' => 'INBOX', 'uidValidity' => 7, 'lock' => ['owner' => 'x'],
+        ]);
+
+        $state = (new MailboxStore($this->dataStore($collection)))->state('svc', 'INBOX');
+
+        $this->assertSame([...MailboxStore::STATE_DEFAULTS, 'uidValidity' => 7], $state);
+    }
+
+    public function testNewMailboxStartsWithDefaultState(): void
+    {
+        $collection = $this->createMock(Collection::class);
+        $collection->expects($this->once())
+            ->method('updateOne')
+            ->with(
+                $this->anything(),
+                $this->callback(fn (array $update): bool => $update['$setOnInsert'] === MailboxStore::STATE_DEFAULTS
+                    && array_intersect_key($update['$set'], MailboxStore::STATE_DEFAULTS) === []),
+                ['upsert' => true],
+            );
+
+        $mailbox = (new CollectionResource('imap', 'svc'))->fromImap(new Mailbox('INBOX', '/', []));
+        (new MailboxStore($this->dataStore($collection)))->upsert('tenant', 'svc', $mailbox);
+    }
+
+    /**
+     * @param array<string, array> $cached
+     * @param string[] $remote
+     */
+    private function harmonizer(array $cached, array $remote): HarmonizationService
+    {
+        $mailboxes = $this->createStub(MailboxStore::class);
+        $mailboxes->method('list')->willReturn($cached);
+        $mailboxes->method('upsert')->willReturnCallback(function (string $tid, string $sid, $collection): void {
+            $this->calls[] = ['upsert', $tid, $sid, $collection->identifier()];
+        });
+        $mailboxes->method('delete')->willReturnCallback(function (string $sid, string $name): void {
+            $this->calls[] = ['mailboxes.delete', $sid, $name];
+        });
+
+        $messages = $this->createStub(MessageStore::class);
+        $messages->method('deleteByMailbox')->willReturnCallback(function (string $sid, string $name): void {
+            $this->calls[] = ['messages.deleteByMailbox', $sid, $name];
+        });
+
+        $files = $this->createStub(MessageFileStore::class);
+        $files->method('deleteByMailbox')->willReturnCallback(function (string $tid, string $sid, string $name): void {
+            $this->calls[] = ['files.deleteByMailbox', $tid, $sid, $name];
+        });
+
+        $service = (new Service())->fromStore(['tid' => 'tenant', 'sid' => 'svc']);
+        $live = new HarmonizationServiceTestLiveStub($service);
+        $live->mailboxes = array_map(static fn (string $name): Mailbox => new Mailbox($name, '/', []), $remote);
+
+        return (new HarmonizationService($mailboxes, $messages, $files))->for($service, $live);
+    }
+
+    private function dataStore(Collection $collection): DataStore
+    {
+        $dataStore = $this->createStub(DataStore::class);
+        $dataStore->method('selectCollection')->willReturn($collection);
+        return $dataStore;
+    }
+}
+
+final class HarmonizationServiceTestLiveStub extends LiveMailService
+{
+    /** @var Mailbox[] */
+    public array $mailboxes = [];
+
+    public function collectionList(?string $location = null, IFilter|null $filter = null, ISort|null $sort = null, string $depth = '*'): Generator
+    {
+        foreach ($this->mailboxes as $mailbox) {
+            yield $mailbox->name() => $mailbox;
+        }
+    }
+}