createStub(DataStore::class); $this->mailboxes = new FakeMailboxStore($dataStore); $this->messages = new FakeMessageStore($dataStore); $this->files = new FakeMessageFileStore('/nonexistent'); $service = (new Service())->fromStore(['tid' => 'tenant', 'sid' => 'svc']); $this->live = new MessageHarmonizationLiveStub($service); $this->live->uidValidity = 7; $this->live->uidNext = 100; $this->harmonizer = (new HarmonizationService( $this->mailboxes, $this->messages, $this->files, new MessageIngestor($this->files, $this->messages, $this->mailboxes), ))->for($service, $this->live); $this->deltas = new MessageDeltaService($this->mailboxes, $this->messages); $this->mailboxes->upsert('tenant', 'svc', (new CollectionResource('imap', 'svc'))->fromImap(new Mailbox('INBOX', '/', []))); } public function testFirstRunIngestsEverythingNewestFirst(): void { $this->live->flags = [1 => ['\\Seen'], 2 => [], 3 => ['\\Flagged']]; $result = $this->harmonizer->harmonizeMessages('INBOX'); $this->assertSame(['status' => 'harmonized', 'reset' => false, 'added' => 3, 'updated' => 0, 'removed' => 0, 'complete' => true], $result); $this->assertSame([[3, 2, 1]], $this->live->fetchBatches); $this->assertSame([1 => ['seen'], 2 => [], 3 => ['flagged']], $this->messages->flags('svc', 'INBOX', 7)); $this->assertSame([1, 2, 3], $this->files->uids(7)); $this->assertSame(262144, $this->live->bodyTextLimit); $state = $this->mailboxes->state('svc', 'INBOX'); $this->assertSame(7, $state['uidValidity']); $this->assertSame(100, $state['uidNext']); $this->assertTrue($state['harmonizationComplete']); $this->assertIsInt($state['harmonizedAt']); $this->assertNull($this->mailboxes->lockOwner); } public function testIncrementalRunAddsUpdatesAndRemoves(): void { $this->live->flags = [1 => ['\\Seen'], 2 => []]; $this->harmonizer->harmonizeMessages('INBOX'); $this->live->flags = [2 => ['\\Seen', '$Forwarded'], 3 => []]; $this->live->fetchBatches = []; $result = $this->harmonizer->harmonizeMessages('INBOX'); $this->assertSame(['status' => 'harmonized', 'reset' => false, 'added' => 1, 'updated' => 1, 'removed' => 1, 'complete' => true], $result); $this->assertSame([[3]], $this->live->fetchBatches); $this->assertSame([2 => ['seen', '$forwarded'], 3 => []], $this->messages->flags('svc', 'INBOX', 7)); $this->assertSame([2, 3], $this->files->uids(7)); } public function testChangedUidValidityDiscardsTheCache(): void { $this->live->flags = [1 => [], 2 => []]; $this->harmonizer->harmonizeMessages('INBOX'); $this->live->uidValidity = 8; $this->live->flags = [1 => []]; $result = $this->harmonizer->harmonizeMessages('INBOX'); $this->assertTrue($result['reset']); $this->assertSame(1, $result['added']); $this->assertSame([], $this->messages->flags('svc', 'INBOX', 7)); $this->assertSame([1 => []], $this->messages->flags('svc', 'INBOX', 8)); $this->assertSame([], $this->files->uids(7)); $this->assertSame(8, $this->mailboxes->state('svc', 'INBOX')['uidValidity']); } public function testHeldLockSkipsTheRun(): void { $this->live->flags = [1 => []]; $this->mailboxes->lockOwner = 'someone-else'; $result = $this->harmonizer->harmonizeMessages('INBOX'); $this->assertSame('skipped', $result['status']); $this->assertSame(0, $this->live->selects); $this->assertSame('someone-else', $this->mailboxes->lockOwner); } public function testMessagesThatCannotBeFetchedLeaveTheMailboxIncomplete(): void { $this->live->flags = [1 => [], 2 => []]; $this->live->unfetchable = [2]; $result = $this->harmonizer->harmonizeMessages('INBOX'); $this->assertSame(1, $result['added']); $this->assertFalse($result['complete']); $this->assertFalse($this->mailboxes->state('svc', 'INBOX')['harmonizationComplete']); } public function testNewMessagesAreFetchedInBatches(): void { $this->live->flags = array_fill_keys(range(1, 450), []); $result = $this->harmonizer->harmonizeMessages('INBOX'); $this->assertSame(450, $result['added']); $this->assertSame([200, 200, 50], array_map('count', $this->live->fetchBatches)); $this->assertSame(450, $this->live->fetchBatches[0][0]); $this->assertSame(1, $this->live->fetchBatches[2][49]); } public function testHarmonizeAllSkipsUnselectableAndIsolatesFailures(): void { $this->live->flags = [1 => []]; $this->live->mailboxList = [ new Mailbox('INBOX', '/', []), new Mailbox('Archive', '/', ['\\Noselect']), new Mailbox('Broken', '/', []), ]; $this->live->failing = ['Broken']; $result = $this->harmonizer->harmonizeAll(); $this->assertSame(['Archive', 'Broken'], $result['mailboxes']['added']); $this->assertSame(['INBOX', 'Broken'], array_keys($result['messages'])); $this->assertSame('harmonized', $result['messages']['INBOX']['status']); $this->assertSame('Mailbox not found on server: Broken', $result['messages']['Broken']); $this->assertNull($this->mailboxes->lockOwner); } public function testEmptySignatureReturnsTheCurrentSignature(): void { $this->live->flags = [1 => [], 2 => [], 3 => []]; $this->harmonizer->harmonizeMessages('INBOX'); $delta = $this->deltas->delta('svc', 'INBOX', ''); $this->assertSame('7:3', $delta->signature); $this->assertSame([[], [], []], $this->changeLists($delta)); } public function testEmptySignatureWhileIncompleteStartsFromZero(): void { $this->live->flags = [1 => [], 2 => []]; $this->live->unfetchable = [2]; $this->harmonizer->harmonizeMessages('INBOX'); $delta = $this->deltas->delta('svc', 'INBOX', ''); $this->assertSame('7:0', $delta->signature); $this->assertSame([['1'], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', '7:0'))); } public function testDeltaReportsAdditionsModificationsAndDeletions(): void { $this->live->flags = [1 => [], 2 => [], 3 => []]; $this->harmonizer->harmonizeMessages('INBOX'); $signature = $this->deltas->delta('svc', 'INBOX', '')->signature; $this->live->flags = [2 => ['\\Seen'], 3 => [], 4 => []]; $this->harmonizer->harmonizeMessages('INBOX'); $delta = $this->deltas->delta('svc', 'INBOX', $signature); $this->assertSame([['4'], ['2'], ['1']], $this->changeLists($delta)); $this->assertSame([[], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', $delta->signature))); } public function testMessagesAddedSinceTheSignatureAreOnlyAdditions(): void { $this->live->flags = [1 => []]; $this->harmonizer->harmonizeMessages('INBOX'); $signature = $this->deltas->delta('svc', 'INBOX', '')->signature; $this->live->flags = [1 => [], 2 => [], 3 => []]; $this->harmonizer->harmonizeMessages('INBOX'); $this->live->flags = [1 => [], 2 => ['\\Flagged']]; $this->harmonizer->harmonizeMessages('INBOX'); // 2: added then flagged -> addition; 3: added then expunged -> not reported $this->assertSame([['2'], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', $signature))); } public function testSignatureThatCannotBeAnsweredIsReset(): void { $this->live->flags = [1 => [], 2 => []]; $this->harmonizer->harmonizeMessages('INBOX'); $old = $this->deltas->delta('svc', 'INBOX', '')->signature; $this->assertSame([['1', '2'], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', 'garbage'))); $this->assertSame([['1', '2'], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', '7:999'))); $this->mailboxes->updateState('svc', 'INBOX', ['purgedSeq' => 2]); $this->assertSame([['1', '2'], [], []], $this->changeLists($this->deltas->delta('svc', 'INBOX', '7:1'))); $this->live->uidValidity = 8; $this->live->flags = [5 => []]; $this->harmonizer->harmonizeMessages('INBOX'); $delta = $this->deltas->delta('svc', 'INBOX', $old); $this->assertSame([['5'], [], []], $this->changeLists($delta)); $this->assertStringStartsWith('8:', $delta->signature); } public function testMailboxThatIsNotCachedHasAnEmptyDelta(): void { $delta = $this->deltas->delta('svc', 'Unknown', '7:1'); $this->assertSame('', $delta->signature); $this->assertSame([[], [], []], $this->changeLists($delta)); } public function testLockIsReleasedWhenTheRunFails(): void { $this->live->uidValidity = null; try { $this->harmonizer->harmonizeMessages('INBOX'); $this->fail('Expected missing UIDVALIDITY to fail the run'); } catch (\RuntimeException) { } $this->assertNull($this->mailboxes->lockOwner); } /** * @return array{0: string[], 1: string[], 2: string[]} */ private function changeLists(Delta $delta): array { return [ json_decode(json_encode($delta->additions), true), json_decode(json_encode($delta->modifications), true), json_decode(json_encode($delta->deletions), true), ]; } } final class MessageHarmonizationLiveStub extends LiveMailService { public ?int $uidValidity = null; public ?int $uidNext = null; /** @var array> */ public array $flags = []; /** @var int[] */ public array $unfetchable = []; public array $fetchBatches = []; public int $selects = 0; public ?int $bodyTextLimit = null; /** @var Mailbox[] */ public array $mailboxList = []; /** @var string[] */ public array $failing = []; public function collectionList(?string $location = null, IFilter|null $filter = null, ISort|null $sort = null, string $depth = '*'): Generator { foreach ($this->mailboxList as $mailbox) { yield $mailbox->name() => $mailbox; } } public function collectionFetch(string $identifier): ?Mailbox { if (in_array($identifier, $this->failing, true)) { return null; } $this->selects++; return new Mailbox($identifier, '/', [], count($this->flags), 0, $this->uidValidity, 0, [], true, $this->uidNext); } public function entityFlags(string $collection): Generator { yield from $this->flags; } public function entityFetch(string $collection, ?FetchOptions $options = null, int ...$uids): Generator { $this->fetchBatches[] = $uids; if (preg_match('/BODY\.PEEK\[TEXT\]<0\.(\d+)>/', (string) $options?->toCommand(), $matches) === 1) { $this->bodyTextLimit = (int) $matches[1]; } foreach ($uids as $uid) { if (in_array($uid, $this->unfetchable, true)) { continue; } $flags = implode(' ', $this->flags[$uid] ?? []); yield $uid => FetchMessageParser::parse("* {$uid} FETCH (UID {$uid} FLAGS ({$flags}))"); } } } final class FakeMailboxStore extends MailboxStore { /** @var array */ public array $documents = []; public ?string $lockOwner = null; public function upsert(string $tenantId, string $serviceId, CollectionResource $collection): void { $name = (string) $collection->identifier(); $this->documents[$name] = [ ...($this->documents[$name] ?? self::STATE_DEFAULTS), ...$collection->toCacheMeta(), ]; } public function fetch(string $serviceId, string $name): ?array { return $this->documents[$name] ?? null; } public function list(string $serviceId): array { return $this->documents; } public function updateState(string $serviceId, string $name, array $state): void { $this->documents[$name] = array_replace($this->documents[$name], array_intersect_key($state, self::STATE_DEFAULTS)); } public function acquireLock(string $serviceId, string $name, string $owner, int $ttl): bool { if ($this->lockOwner !== null && $this->lockOwner !== $owner) { return false; } $this->lockOwner = $owner; return true; } public function releaseLock(string $serviceId, string $name, string $owner): void { if ($this->lockOwner === $owner) { $this->lockOwner = null; } } public function reserveSequence(string $serviceId, string $name, int $count = 1): int { $this->documents[$name]['changeSeq'] += $count; return $this->documents[$name]['changeSeq'] - $count + 1; } public function state(string $serviceId, string $name): array { return array_replace(self::STATE_DEFAULTS, array_intersect_key($this->documents[$name] ?? [], self::STATE_DEFAULTS)); } } final class FakeMessageStore extends MessageStore { /** @var array, addedSeq: int, modSeq: int, removedSeq: ?int}>> by UIDVALIDITY and UID */ private array $documents = []; public function upsert(string $tenantId, string $serviceId, int $uidValidity, EntityResource $entity, int $seq): void { $meta = $entity->toCacheMeta(); $existing = $this->documents[$uidValidity][$meta['uid']] ?? null; $this->documents[$uidValidity][$meta['uid']] = [ 'flags' => $meta['flags'], 'addedSeq' => $existing['addedSeq'] ?? $seq, 'modSeq' => $seq, 'removedSeq' => null, ]; ksort($this->documents[$uidValidity]); } public function flags(string $serviceId, string $mailbox, int $uidValidity): array { $flags = []; foreach ($this->documents[$uidValidity] ?? [] as $uid => $document) { if ($document['removedSeq'] === null) { $flags[$uid] = $document['flags']; } } return $flags; } public function updateFlags(string $serviceId, string $mailbox, int $uidValidity, int $uid, array $flags, int $seq): void { $this->documents[$uidValidity][$uid]['flags'] = array_values($flags); $this->documents[$uidValidity][$uid]['modSeq'] = $seq; } public function tombstone(string $serviceId, string $mailbox, int $uidValidity, int $seq, int ...$uids): void { foreach ($uids as $uid) { $this->documents[$uidValidity][$uid]['removedSeq'] = $seq; $this->documents[$uidValidity][$uid]['modSeq'] = $seq; $this->documents[$uidValidity][$uid]['flags'] = []; } } public function changes(string $serviceId, string $mailbox, int $uidValidity, int $since): array { $changes = ['additions' => [], 'modifications' => [], 'deletions' => []]; foreach ($this->documents[$uidValidity] ?? [] as $uid => $document) { if ($document['removedSeq'] !== null) { if ($document['addedSeq'] <= $since && $document['removedSeq'] > $since) { $changes['deletions'][] = $uid; } } elseif ($document['addedSeq'] > $since) { $changes['additions'][] = $uid; } elseif ($document['modSeq'] > $since) { $changes['modifications'][] = $uid; } } return $changes; } public function deleteByMailbox(string $serviceId, string $mailbox): void { $this->documents = []; } } final class FakeMessageFileStore extends MessageFileStore { /** @var array> content by UIDVALIDITY and UID */ private array $files = []; public function write(string $tenantId, string $serviceId, string $mailbox, int $uidValidity, int $uid, array $content): void { $this->files[$uidValidity][$uid] = $content; } public function delete(string $tenantId, string $serviceId, string $mailbox, int $uidValidity, int ...$uids): void { foreach ($uids as $uid) { unset($this->files[$uidValidity][$uid]); } } public function deleteByMailbox(string $tenantId, string $serviceId, string $mailbox): void { $this->files = []; } /** @return int[] */ public function uids(int $uidValidity): array { $uids = array_keys($this->files[$uidValidity] ?? []); sort($uids); return $uids; } }