Nodarx/provider_imap
1
0
Fork 0
generated from Nodarx/template
Code Issues 1 Pull Requests 5 Actions Packages Projects Releases Wiki Activity

feat: harmonize mailbox messages

Browse Source 
Signed-off-by: Sebastian Krupinski <krupinski01@gmail.com>
This commit is contained in:
Sebastian Krupinski
2026-10-05 20:34:02 -04:00
parent b02ca63208
commit 65fdb23fa4
12 changed files with 879 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/Service/Cache/HarmonizationService.php
+143
View File
@@ -9,13 +9,17 @@ declare(strict_types=1);
namespace KTXM\ProviderImap\Service\Cache;
use KTXM\ProviderImap\Client\Protocol\Command\Argument\FetchOptions;
use KTXM\ProviderImap\Providers\CollectionResource;
use KTXM\ProviderImap\Providers\EntityResource;
use KTXM\ProviderImap\Providers\MessageProperties;
use KTXM\ProviderImap\Providers\Service;
use KTXM\ProviderImap\Service\Live\LiveMailService;
use KTXM\ProviderImap\Stores\MailboxStore;
use KTXM\ProviderImap\Stores\MessageFileStore;
use KTXM\ProviderImap\Stores\MessageStore;
use LogicException;
use RuntimeException;
/**
* Brings the cache of one service in line with its IMAP server.
@@ -25,6 +29,12 @@ use LogicException;
*/
class HarmonizationService
{
/** Messages fetched per UID FETCH when ingesting; each is cached as it streams in */
public const INGEST_BATCH_SIZE = 200;
/** Lifetime of a mailbox harmonization lock in seconds; long enough for a large first run */
public const LOCK_TTL = 900;
private ?Service $service = null;
private ?LiveMailService $live = null;
@@ -32,6 +42,7 @@ class HarmonizationService
private readonly MailboxStore $mailboxStore,
private readonly MessageStore $messageStore,
private readonly MessageFileStore $fileStore,
private readonly MessageIngestor $ingestor,
) {}
/**
@@ -87,6 +98,138 @@ class HarmonizationService
return $result;
}
/**
* Harmonize the messages of one mailbox with the server.
*
* Ingests messages that are not cached yet (newest first), updates changed
* flags and removes messages expunged on the server. A changed UIDVALIDITY
* discards the mailbox's cache first. Never waits: when another run holds the
* mailbox lock, nothing is done.
*
* @return array{status: 'harmonized'|'skipped', reset: bool, added: int, updated: int, removed: int, complete: bool}
*/
public function harmonizeMessages(string $mailbox): array
{
[$service, $live] = $this->bound();
$tenantId = (string) $service->tenantIdentifier();
$serviceId = (string) $service->identifier();
$result = ['status' => 'skipped', 'reset' => false, 'added' => 0, 'updated' => 0, 'removed' => 0, 'complete' => false];
// the lock lives on the mailbox document, so it has to exist first
if ($this->mailboxStore->fetch($serviceId, $mailbox) === null) {
$this->harmonizeMailboxes();
if ($this->mailboxStore->fetch($serviceId, $mailbox) === null) {
throw new RuntimeException("Mailbox not found on server: {$mailbox}");
}
}
$owner = bin2hex(random_bytes(8));
if (!$this->mailboxStore->acquireLock($serviceId, $mailbox, $owner, self::LOCK_TTL)) {
return $result;
}
try {
$state = $this->mailboxStore->state($serviceId, $mailbox);
$selected = $live->collectionFetch($mailbox)
?? throw new RuntimeException("Mailbox not found on server: {$mailbox}");
$uidValidity = $selected->uidValidity()
?? throw new RuntimeException("Server did not report UIDVALIDITY for mailbox: {$mailbox}");
// a new UIDVALIDITY invalidates every cached UID of the mailbox
if ($state['uidValidity'] !== null && (int) $state['uidValidity'] !== $uidValidity) {
$this->messageStore->deleteByMailbox($serviceId, $mailbox);
$this->fileStore->deleteByMailbox($tenantId, $serviceId, $mailbox);
$result['reset'] = true;
}
$cachedFlags = $this->messageStore->flags($serviceId, $mailbox, $uidValidity);
// stream every UID and its flags from the server: cached messages get their flags
// compared (and are dropped from $cachedFlags), unknown UIDs are collected as new.
// No other IMAP command may run until the stream is consumed.
$new = [];
foreach ($live->entityFlags($mailbox) as $uid => $flags) {
if (!isset($cachedFlags[$uid])) {
$new[] = $uid;
continue;
}
$flags = MessageProperties::normalizeFlags($flags);
if (!self::sameFlags($flags, $cachedFlags[$uid])) {
$this->messageStore->updateFlags($serviceId, $mailbox, $uidValidity, $uid, $flags);
$result['updated']++;
}
unset($cachedFlags[$uid]);
}
// new: newest first so recent mail is available soonest
rsort($new);
$missing = [];
foreach (array_chunk($new, self::INGEST_BATCH_SIZE) as $batch) {
$ingested = $this->ingestBatch($tenantId, $serviceId, $mailbox, $uidValidity, $batch);
$result['added'] += count($ingested);
array_push($missing, ...array_diff($batch, $ingested));
}
// expunged: cached UIDs the server did not list; meta documents first, then files
$expunged = array_keys($cachedFlags);
if ($expunged !== []) {
$this->messageStore->delete($serviceId, $mailbox, $uidValidity, ...$expunged);
$this->fileStore->delete($tenantId, $serviceId, $mailbox, $uidValidity, ...$expunged);
$result['removed'] = count($expunged);
}
// UIDs that could not be fetched (e.g. expunged meanwhile) are retried next run
$result['complete'] = $missing === [];
$result['status'] = 'harmonized';
$this->mailboxStore->updateState($serviceId, $mailbox, [
'uidValidity' => $uidValidity,
'uidNext' => $selected->uidNext(),
'highestModSeq' => $selected->highestModSeq(),
'harmonizedAt' => time(),
'harmonizationComplete' => $result['complete'],
]);
} finally {
$this->mailboxStore->releaseLock($serviceId, $mailbox, $owner);
}
return $result;
}
/**
* Fetch a batch of messages and cache each one as it streams in.
*
* @param int[] $uids
* @return int[] UIDs that were cached
*/
private function ingestBatch(string $tenantId, string $serviceId, string $mailbox, int $uidValidity, array $uids): array
{
[$service, $live] = $this->bound();
$options = FetchOptions::message()->withBodyText(MessageIngestor::BODY_TEXT_LIMIT);
$ingested = [];
foreach ($live->entityFetch($mailbox, $options, ...$uids) as $message) {
$entity = (new EntityResource($service->provider(), $serviceId))->fromImap($message, $mailbox);
array_push($ingested, ...$this->ingestor->ingest($tenantId, $serviceId, $uidValidity, $entity));
}
return $ingested;
}
/**
* @param string[] $left
* @param string[] $right
*/
private static function sameFlags(array $left, array $right): bool
{
sort($left);
sort($right);
return $left === $right;
}
/**
* Remove a mailbox and everything cached for it: meta documents first, then files, then the mailbox.
*/
lib/Service/Live/LiveMailService.php
+78 -17
View File
@@ -65,6 +65,8 @@ class LiveMailService
private ?ImapClient $imapClient = null;
private ?SmtpClient $smtpClient = null;
/** Result of the last SELECT on the current connection, reused while still selected */
private ?Mailbox $selected = null;
public function __construct(
private readonly Service $service,
@@ -200,16 +202,25 @@ class LiveMailService
*/
public function collectionFetch(string $identifier): ?Mailbox
{
// LIST-STATUS (RFC 5819) returns the status with the LIST response in one round trip
$listStatus = $this->imapClient()->hasCapability('LIST-STATUS');
$command = $listStatus
? new ListCommand('', $identifier, null, ListReturnOptions::status(...self::DEFAULT_MAILBOX_STATUS_ITEMS))
: new ListCommand('', $identifier);
// retrieve mailbox from remote
$mailbox = iterator_to_array($this->imapClient()->perform(new ListCommand('', $identifier, null, ListReturnOptions::status(...self::DEFAULT_MAILBOX_STATUS_ITEMS))));
$mailbox = iterator_to_array($this->imapClient()->perform($command));
if (empty($mailbox)) {
return null;
}
$mailbox = reset($mailbox);
// enrich with STATUS
$status = $this->imapClient()->perform(new StatusCommand($mailbox->name(), self::DEFAULT_MAILBOX_STATUS_ITEMS));
$mailbox = $mailbox->fromStatus($status);
// enrich with STATUS when LIST could not provide it
if (!$listStatus && $mailbox->isSelectable()) {
$status = $this->imapClient()->perform(new StatusCommand($mailbox->name(), self::DEFAULT_MAILBOX_STATUS_ITEMS));
$mailbox = $mailbox->fromStatus($status);
}
return $mailbox;
}
@@ -270,7 +281,7 @@ class LiveMailService
$nativeFilter = $this->buildEntitySearchCriteria($filter);
$nativeSort = $sort !== null ? $this->entitySortCriteria($sort) : [];
$this->imapClient()->perform(new SelectCommand($collection, true));
$this->select($collection);
$rfc5258 = $this->imapClient()->hasCapability('SORT');
$uids = [];
@@ -298,6 +309,31 @@ class LiveMailService
return $this->entityApplyRange($uids, $range);
}
/**
* Stream the flags of every message in a mailbox.
*
* The stream also lists every UID that exists in the mailbox.
*
* @return Generator<int, list<string>> IMAP flags keyed by UID
*/
public function entityFlags(string $collection): Generator
{
// fresh SELECT: the message count decides whether "1:*" may be sent
$mailbox = $this->select($collection, refresh: true);
// "1:*" on an empty mailbox is rejected by some servers
if ($mailbox->messages() === 0) {
return;
}
foreach ($this->imapClient()->perform(new FetchManyCommand(
MessageTarget::uid('1:*'),
FetchOptions::of('FLAGS'),
)) as $message) {
yield $message->uid() => $message->flags();
}
}
/**
* Determine which of the given UIDs exist in a mailbox.
*
@@ -309,7 +345,7 @@ class LiveMailService
return [];
}
$this->imapClient()->perform(new SelectCommand($collection, true));
$this->select($collection);
return $this->imapClient()->perform(new SearchCommand(
SearchCriteriaBuilder::create()->uid(SequenceSet::items(...$uids)),
@@ -329,9 +365,10 @@ class LiveMailService
// fast path: fetch all messages without filtering, sorting or pagination
if ($filter === null && $sort === null && $range === null) {
$mailbox = $this->imapClient()->perform(new SelectCommand($collection, true));
$mailbox = $this->select($collection, refresh: true);
if ($mailbox === null) {
// "1:*" on an empty mailbox is rejected by some servers
if ($mailbox->messages() === 0) {
return [];
}
@@ -366,7 +403,7 @@ class LiveMailService
}
$options ??= FetchOptions::message()->withBodyText();
$this->imapClient()->perform(new SelectCommand($collection, true));
$this->select($collection);
$request = new FetchManyCommand(
MessageTarget::uid(SequenceSet::items(...array_values($uids))),
@@ -391,7 +428,7 @@ class LiveMailService
*/
public function entityDownload(string $collection, int $uid, ?string $partId = null): BinaryResource
{
$this->imapClient()->perform(new SelectCommand($collection, true));
$this->select($collection);
$encoding = null;
@@ -550,7 +587,7 @@ class LiveMailService
return;
}
$this->imapClient()->perform(new SelectCommand($collection, false));
$this->select($collection, readOnly: false);
$this->imapClient()->perform(new StoreCommand(
MessageTarget::uid(SequenceSet::items(...array_values($uids))),
$flags,
@@ -569,7 +606,7 @@ class LiveMailService
$target = MessageTarget::uid(SequenceSet::items(...array_values($uids)));
$this->imapClient()->perform(new SelectCommand($collection, false));
$this->select($collection, readOnly: false);
$this->imapClient()->perform(new StoreCommand($target, ['\\Deleted'], '+'));
$this->imapClient()->perform(new ExpungeCommand($target));
@@ -583,7 +620,7 @@ class LiveMailService
return;
}
$this->imapClient()->perform(new SelectCommand($collection, false));
$this->select($collection, readOnly: false);
$flagsToAdd = $this->normalizeFlags($flagsToAdd);
$flagsToRemove = $this->normalizeFlags($flagsToRemove);
@@ -615,13 +652,13 @@ class LiveMailService
// if MOVE is supported, use it; otherwise, fall back to COPY + EXPUNGE
if ($rfc6851) {
$this->imapClient()->perform(new SelectCommand($sourceCollection, false));
$this->select($sourceCollection, readOnly: false);
$response = $this->imapClient()->perform(new MoveCommand(
MessageTarget::uid(SequenceSet::items(...array_values($uids))),
$targetCollection,
));
} else {
$this->imapClient()->perform(new SelectCommand($sourceCollection, false));
$this->select($sourceCollection, readOnly: false);
$response = $this->imapClient()->perform(new CopyCommand(
MessageTarget::uid(SequenceSet::items(...array_values($uids))),
$targetCollection,
@@ -657,7 +694,7 @@ class LiveMailService
return [];
}
$this->imapClient()->perform(new SelectCommand($sourceCollection, false));
$this->select($sourceCollection, readOnly: false);
$response = $this->imapClient()->perform(new CopyCommand(
MessageTarget::uid(SequenceSet::items(...array_values($uids))),
$targetCollection,
@@ -1143,4 +1180,28 @@ class LiveMailService
}
return $normalized;
}
/**
* Select a mailbox, reusing the current selection when possible.
*
* The current selection is reused when the same mailbox is still selected on the
* session and its access mode suffices (read-write covers read-only). A refresh
* forces a new SELECT, e.g. when an up-to-date message count is needed.
*/
private function select(string $collection, bool $readOnly = true, bool $refresh = false): Mailbox
{
if (!$refresh
&& $this->selected !== null
&& $this->selected->name() === $collection
&& $this->imapClient()->session()->selectedMailbox() === $collection
&& ($readOnly || !$this->selected->readOnly())
) {
return $this->selected;
}
$this->selected = null;
$this->selected = $this->imapClient()->perform(new SelectCommand($collection, $readOnly));
return $this->selected;
}
}