feat: import people

Signed-off-by: Sebastian Krupinski <krupinski01@gmail.com>
This commit is contained in:
2026-06-22 21:59:27 -04:00
parent cee6c2924c
commit 5fbc217edb
19 changed files with 1321 additions and 73 deletions
+107 -24
View File
@@ -23,7 +23,10 @@ use KTXF\Resource\Identifier\ResourceIdentifier;
use KTXF\Resource\Identifier\ResourceIdentifiers;
use KTXF\Resource\Identifier\ServiceIdentifier;
use KTXF\Routing\Attributes\AuthenticatedRoute;
use KTXM\PeopleManager\Import\ImportOptions;
use KTXM\PeopleManager\Import\ImportService;
use KTXM\PeopleManager\Manager;
use KTXM\PeopleManager\Stream\ExpectedTotal;
use Psr\Log\LoggerInterface;
use Throwable;
@@ -50,6 +53,7 @@ class DefaultController extends ControllerAbstract {
private readonly SessionTenant $tenantIdentity,
private readonly SessionIdentity $userIdentity,
private readonly Manager $manager,
private readonly ImportService $importService,
private readonly LoggerInterface $logger,
) {}
@@ -153,7 +157,8 @@ class DefaultController extends ControllerAbstract {
'entity.delete' => $this->entityDelete($tenantId, $userId, $data),
'entity.move' => $this->entityMove($tenantId, $userId, $data),
'entity.copy' => $this->entityCopy($tenantId, $userId, $data),
'entity.import' => $this->entityImport($tenantId, $userId, $data, $version, $transaction),
default => throw new InvalidArgumentException(self::ERR_INVALID_OPERATION . $operation)
};
}
@@ -540,39 +545,117 @@ class DefaultController extends ControllerAbstract {
$sources = null;
}
$filter = $data['filter'] ?? null;
$sort = $data['sort'] ?? null;
$range = $data['range'] ?? null;
$entityGenerator = $this->manager->entityListStream($tenantId, $userId, $sources, $filter, $sort, $range);
$logger = $this->logger;
$entities = $this->manager->entityListStream($tenantId, $userId, $sources, $filter, $sort, $range);
$responseGenerator = (function () use ($entityGenerator, $version, $transaction, $logger): \Generator {
yield ['type' => 'control', 'status' => 'start', 'version' => $version, 'transaction' => $transaction];
return new StreamedNdJsonResponse(
$this->streamEnvelope($entities, $version, $transaction),
1,
200,
['Content-Type' => 'application/json'],
);
}
$total = 0;
/**
* Import vCards into a collection, streaming one NDJSON event per contact.
*
* Request data: { target: "provider:service:collection", data: "<raw vCard string>", options?: {...} }
*/
private function entityImport(string $tenantId, string $userId, array $data, int $version, string $transaction): StreamedNdJsonResponse {
if (!isset($data['target'])) {
throw new InvalidArgumentException(self::ERR_MISSING_TARGET);
}
if (!is_string($data['target'])) {
throw new InvalidArgumentException(self::ERR_INVALID_TARGET);
}
if (!isset($data['data']) || !is_string($data['data']) || trim($data['data']) === '') {
throw new InvalidArgumentException('Invalid parameter: data must be a non-empty string');
}
$target = ResourceIdentifier::fromString($data['target']);
if (!$target instanceof CollectionIdentifier) {
throw new InvalidArgumentException('Invalid parameter: target must be provider:service:collection');
}
$options = ImportOptions::fromArray($data['options'] ?? []);
// Spill the payload to a temp file and release the in-memory copy before the
// (potentially long) streaming parse, so peak memory stays at one contact object.
$tempFile = tmpfile();
if ($tempFile === false) {
throw new \RuntimeException('Unable to allocate temporary file for import');
}
fwrite($tempFile, $data['data']);
unset($data);
rewind($tempFile);
$events = $this->importService->import($tempFile, $target, $options, $tenantId, $userId);
$frames = $this->streamEnvelope($events, $version, $transaction);
// Stream the envelope, releasing the spilled temp file once it is fully
// drained (or the client disconnects) so it never outlives its stream.
$response = (function () use ($frames, $tempFile): \Generator {
try {
foreach ($entityGenerator as $entity) {
if (!$entity instanceof JsonSerializable) {
continue;
}
yield [
'type' => 'data',
'data' => $entity->jsonSerialize()
];
$total++;
}
} catch (\Throwable $t) {
$logger->error('Error streaming entities', ['exception' => $t]);
yield ['type' => 'error', 'message' => $t->getMessage()];
return;
yield from $frames;
} finally {
fclose($tempFile);
}
yield ['type' => 'control', 'status' => 'end', 'total' => $total];
})();
return new StreamedNdJsonResponse($responseGenerator, 1, 200, ['Content-Type' => 'application/json']);
return new StreamedNdJsonResponse($response, 1, 200);
}
/**
* Wrap a generator of JsonSerializable domain objects in the canonical NDJSON
* stream envelope shared by every streaming operation:
*
* control:start {version, transaction, total?} — total? = expected count
* data {data} — one per domain object
* error {message} — on failure, then stop
* control:end {total} — total = objects emitted
*
* If the generator leads with an {@see ExpectedTotal} event, its value is
* folded into the start frame's `total` (the progress denominator) rather
* than emitted as a data frame.
*
* @param \Generator<\JsonSerializable> $items
*/
private function streamEnvelope(\Generator $items, int $version, string $transaction): \Generator {
// Peek the first event: an expected-total marker rides on the start frame.
$expected = null;
$items->rewind();
if ($items->valid() && $items->current() instanceof ExpectedTotal) {
$expected = $items->current()->expectedTotal();
$items->next();
}
$start = ['type' => 'control', 'status' => 'start', 'version' => $version, 'transaction' => $transaction];
if ($expected !== null) {
$start['total'] = $expected;
}
yield $start;
$total = 0;
try {
for (; $items->valid(); $items->next()) {
$item = $items->current();
if (!$item instanceof \JsonSerializable) {
continue;
}
yield ['type' => 'data', 'data' => $item->jsonSerialize()];
$total++;
}
} catch (\Throwable $t) {
$this->logger->error('Error streaming response', ['exception' => $t]);
yield ['type' => 'error', 'message' => $t->getMessage()];
return;
}
yield ['type' => 'control', 'status' => 'end', 'total' => $total];
}
private function entityFetch(string $tenantId, string $userId, array $data): mixed {
+38
View File
@@ -0,0 +1,38 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: Sebastian Krupinski <krupinski01@gmail.com>
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace KTXM\PeopleManager\Import;
use KTXM\PeopleManager\Stream\ExpectedTotal;
/**
* Carries the total number of vCards discovered in the source.
* Yielded once, before any object events, when counting is enabled.
*
* The stream envelope consumes this as the `control:start` total (the progress
* denominator); it is never emitted as a `data` frame, so it carries no payload
* of its own.
*/
final readonly class ImportCountEvent implements ImportEvent, ExpectedTotal {
public function __construct(
public int $total,
) {}
public function expectedTotal(): int {
return $this->total;
}
/**
* @return array{total: int}
*/
public function jsonSerialize(): array {
return ['total' => $this->total];
}
}
+17
View File
@@ -0,0 +1,17 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: Sebastian Krupinski <krupinski01@gmail.com>
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace KTXM\PeopleManager\Import;
enum ImportDisposition: string {
case Created = 'created';
case Updated = 'updated';
case Exists = 'exists';
case Error = 'error';
}
+21
View File
@@ -0,0 +1,21 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: Sebastian Krupinski <krupinski01@gmail.com>
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace KTXM\PeopleManager\Import;
use JsonSerializable;
/**
* Marker interface for events yielded by the contact import generator.
*
* Each event serializes to a flat NDJSON frame (no nested envelope); the
* controller appends the transaction id and streams it as one line.
*/
interface ImportEvent extends JsonSerializable {
}
+40
View File
@@ -0,0 +1,40 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: Sebastian Krupinski <krupinski01@gmail.com>
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace KTXM\PeopleManager\Import;
/**
* Yielded once per vCard processed, carrying its disposition and any errors.
*/
final readonly class ImportObjectEvent implements ImportEvent {
/**
* @param list<string> $errors
*/
public function __construct(
public ?string $identifier,
public ImportDisposition $disposition,
public array $errors = [],
) {}
public function isError(): bool {
return $this->disposition === ImportDisposition::Error;
}
/**
* @return array{identifier: ?string, disposition: string, errors: list<string>}
*/
public function jsonSerialize(): array {
return [
'identifier' => $this->identifier,
'disposition' => $this->disposition->value,
'errors' => $this->errors,
];
}
}
+77
View File
@@ -0,0 +1,77 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: Sebastian Krupinski <krupinski01@gmail.com>
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace KTXM\PeopleManager\Import;
use InvalidArgumentException;
/**
* Configuration for a contact import run.
*/
final class ImportOptions {
public const ERROR_CONTINUE = 0;
public const ERROR_FAIL = 1;
public const ERROR_OPTIONS = [self::ERROR_CONTINUE, self::ERROR_FAIL];
/** Overwrite an existing entity when the same UID is already present. */
private bool $supersede = false;
/** Emit an ImportCountEvent (discovered total) before the object stream. */
private bool $counts = true;
/** How to handle per-object errors. */
private int $errors = self::ERROR_CONTINUE;
public function getSupersede(): bool {
return $this->supersede;
}
public function setSupersede(bool $value): void {
$this->supersede = $value;
}
public function getCounts(): bool {
return $this->counts;
}
public function setCounts(bool $value): void {
$this->counts = $value;
}
public function getErrors(): int {
return $this->errors;
}
public function setErrors(int $value): void {
if (!in_array($value, self::ERROR_OPTIONS, true)) {
throw new InvalidArgumentException('Invalid errors option specified');
}
$this->errors = $value;
}
/**
* Build options from the raw `options` array carried in the request.
*
* @param array<string,mixed> $data
*/
public static function fromArray(array $data): self {
$options = new self();
if (isset($data['supersede'])) {
$options->setSupersede((bool)$data['supersede']);
}
if (isset($data['counts'])) {
$options->setCounts((bool)$data['counts']);
}
if (isset($data['errors'])) {
$options->setErrors((int)$data['errors']);
}
return $options;
}
}
+372
View File
@@ -0,0 +1,372 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: Sebastian Krupinski <krupinski01@gmail.com>
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace KTXM\PeopleManager\Import;
use Generator;
use InvalidArgumentException;
use KTXF\People\Entity as E;
use KTXF\People\Entity\Property as P;
use KTXF\Resource\Identifier\CollectionIdentifier;
use KTXM\PeopleManager\Manager;
use Sabre\VObject\Component\VCard as VCardComponent;
use Sabre\VObject\ParseException;
use Sabre\VObject\Parser\MimeDir;
use Sabre\VObject\Property;
use Sabre\VObject\Splitter\VCard as VCardSplitter;
use Throwable;
/**
* Streams vCards from a file resource, maps each to a people entity, and creates
* it in the target collection — yielding one ImportEvent per card.
*
* Memory profile: one vCard is held at a time (sabre's streaming splitter), the
* generator keeps no running collection, and the discovered count is obtained by a
* cheap byte scan rather than a second parse. See modules/people/.ideas/vcf-import.md.
*
* v1 scope: every card is created (disposition `created`) or fails (`error`). The
* vCard UID is mapped to the entity `urid` so future supersede/exists detection is
* possible, but the local provider keys entities on a fresh server id and exposes no
* UID lookup, so supersede/updated/exists are deferred. Individual ORG affiliation
* mapping is also deferred (the ORG property on individual cards is not yet mapped).
*/
class ImportService {
public function __construct(
private readonly Manager $manager,
) {}
/**
* @param resource $source readable, seekable file resource containing one or more vCards
*
* @return Generator<int, ImportEvent>
*/
public function import($source, CollectionIdentifier $target, ImportOptions $options, string $tenantId, string $userId): Generator {
if (!is_resource($source)) {
throw new InvalidArgumentException('Invalid import source: must be a file resource');
}
// Discovered count — cheap O(1)-memory byte scan, no second parse.
if ($options->getCounts()) {
yield new ImportCountEvent($this->countCards($source));
}
rewind($source);
$splitter = new VCardSplitter($source, MimeDir::OPTION_FORGIVING | MimeDir::OPTION_IGNORE_INVALID_LINES);
while (true) {
try {
$vCard = $splitter->getNext();
} catch (ParseException $e) {
// The parser position is unreliable after a parse error, so stop here.
yield new ImportObjectEvent(null, ImportDisposition::Error, ['Malformed vCard: ' . $e->getMessage()]);
return;
}
if ($vCard === null) {
break;
}
$uid = isset($vCard->UID) ? (string)$vCard->UID : null;
try {
$properties = $this->mapVCard($vCard);
$this->manager->entityCreate($tenantId, $userId, $target, $properties);
yield new ImportObjectEvent($uid, ImportDisposition::Created);
} catch (Throwable $e) {
yield new ImportObjectEvent($uid, ImportDisposition::Error, [$e->getMessage()]);
if ($options->getErrors() === ImportOptions::ERROR_FAIL) {
return;
}
}
unset($vCard);
}
}
/**
* Count vCards by scanning for BEGIN:VCARD lines — no parsing, O(1) memory.
*
* @param resource $source
*/
private function countCards($source): int {
rewind($source);
$count = 0;
while (($line = fgets($source)) !== false) {
if (stripos($line, 'BEGIN:VCARD') === 0) {
$count++;
}
}
rewind($source);
return $count;
}
/**
* Map a sabre vCard to an entity property array (the shape entity.create consumes).
*
* @return array<string,mixed>
*/
private function mapVCard(VCardComponent $vCard): array {
$kind = isset($vCard->KIND) ? strtolower((string)$vCard->KIND) : 'individual';
return match ($kind) {
'org', 'organization' => $this->mapOrganization($vCard)->jsonSerialize(),
'group' => $this->mapGroup($vCard)->jsonSerialize(),
default => $this->mapIndividual($vCard)->jsonSerialize(),
};
}
private function mapIndividual(VCardComponent $vCard): E\IndividualObject {
$object = new E\IndividualObject();
$object->urid = $this->uid($vCard);
$object->label = $this->text($vCard, 'FN');
// Structured name: N = family;given;additional;prefix;suffix
if (isset($vCard->N)) {
$parts = $vCard->N->getParts();
$object->names->family = $this->part($parts, 0);
$object->names->given = $this->part($parts, 1);
$object->names->additional = $this->part($parts, 2);
$object->names->prefix = $this->part($parts, 3);
$object->names->suffix = $this->part($parts, 4);
}
foreach ($vCard->select('EMAIL') as $prop) {
$email = new P\EmailObject();
$email->address = (string)$prop;
$email->context = $this->firstType($prop);
$object->emails->add($email, $this->key());
}
foreach ($vCard->select('TEL') as $prop) {
$phone = new P\PhoneObject();
$phone->number = (string)$prop;
$phone->context = $this->firstType($prop);
$object->phones->add($phone, $this->key());
}
foreach ($vCard->select('ADR') as $prop) {
$object->physicalLocations->add($this->address(new P\PhysicalLocationObject(), $prop), $this->key());
}
foreach ($vCard->select('URL') as $prop) {
$location = new P\VirtualLocationObject();
$location->location = (string)$prop;
$location->context = $this->firstType($prop);
$object->virtualLocations->add($location, $this->key());
}
foreach ($vCard->select('NOTE') as $prop) {
$note = new P\NoteObject();
$note->content = (string)$prop;
$object->notes->add($note, $this->key());
}
foreach ($vCard->select('TITLE') as $prop) {
$title = new P\TitleObject();
$title->kind = P\TitleTypes::Title;
$title->label = (string)$prop;
$object->titles->add($title, $this->key());
}
foreach ($vCard->select('ROLE') as $prop) {
$title = new P\TitleObject();
$title->kind = P\TitleTypes::Role;
$title->label = (string)$prop;
$object->titles->add($title, $this->key());
}
if (isset($vCard->BDAY)) {
$this->anniversary($object, P\AnniversaryTypes::Birth, (string)$vCard->BDAY);
}
if (isset($vCard->ANNIVERSARY)) {
$this->anniversary($object, P\AnniversaryTypes::Nuptial, (string)$vCard->ANNIVERSARY);
}
foreach ($this->categories($vCard) as $tag) {
$object->tags->add($tag);
}
return $object;
}
private function mapOrganization(VCardComponent $vCard): E\OrganizationObject {
$object = new E\OrganizationObject();
$object->urid = $this->uid($vCard);
$object->label = $this->text($vCard, 'FN') ?? $this->text($vCard, 'ORG');
$object->names->full = $object->label;
$object->names->sort = $this->text($vCard, 'SORT-STRING');
foreach ($vCard->select('EMAIL') as $prop) {
$email = new P\EmailObject();
$email->address = (string)$prop;
$email->context = $this->firstType($prop);
$object->emails->add($email, $this->key());
}
foreach ($vCard->select('TEL') as $prop) {
$phone = new P\PhoneObject();
$phone->number = (string)$prop;
$phone->context = $this->firstType($prop);
$object->phones->add($phone, $this->key());
}
foreach ($vCard->select('ADR') as $prop) {
$object->physicalLocations->add($this->address(new P\PhysicalLocationObject(), $prop), $this->key());
}
foreach ($vCard->select('URL') as $prop) {
$location = new P\VirtualLocationObject();
$location->location = (string)$prop;
$location->context = $this->firstType($prop);
$object->virtualLocations->add($location, $this->key());
}
foreach ($vCard->select('NOTE') as $prop) {
$note = new P\NoteObject();
$note->content = (string)$prop;
$object->notes->add($note, $this->key());
}
return $object;
}
private function mapGroup(VCardComponent $vCard): E\GroupObject {
$object = new E\GroupObject();
$object->urid = $this->uid($vCard);
$object->label = $this->text($vCard, 'FN');
$object->names->full = $object->label;
foreach ($vCard->select('MEMBER') as $prop) {
$member = new P\MemberObject();
// MEMBER values are URIs, commonly urn:uuid:<id>.
$member->entityId = preg_replace('/^urn:uuid:/i', '', (string)$prop);
$object->members->add($member, $this->key());
}
foreach ($vCard->select('URL') as $prop) {
$location = new P\VirtualLocationObject();
$location->location = (string)$prop;
$location->context = $this->firstType($prop);
$object->virtualLocations->add($location, $this->key());
}
foreach ($vCard->select('NOTE') as $prop) {
$note = new P\NoteObject();
$note->content = (string)$prop;
$object->notes->add($note, $this->key());
}
return $object;
}
// ==================== Helpers ====================
/**
* Populate a physical-location object from an ADR property.
* ADR = pobox;ext(unit);street;locality;region;code;country
*/
private function address(P\PhysicalLocationObject $location, Property $prop): P\PhysicalLocationObject {
$parts = $prop->getParts();
$location->box = $this->part($parts, 0);
$location->unit = $this->part($parts, 1);
$location->street = $this->part($parts, 2);
$location->locality = $this->part($parts, 3);
$location->region = $this->part($parts, 4);
$location->code = $this->part($parts, 5);
$location->country = $this->part($parts, 6);
$location->context = $this->firstType($prop);
return $location;
}
private function anniversary(E\IndividualObject $object, P\AnniversaryTypes $type, string $value): void {
$date = $this->date($value);
if ($date === null) {
return;
}
$anniversary = new P\AnniversaryObject();
$anniversary->type = $type;
$anniversary->when = $date;
$object->anniversaries->add($anniversary);
}
private function uid(VCardComponent $vCard): ?string {
return isset($vCard->UID) ? (string)$vCard->UID : null;
}
private function text(VCardComponent $vCard, string $name): ?string {
if (!isset($vCard->$name)) {
return null;
}
$value = trim((string)$vCard->$name);
return $value === '' ? null : $value;
}
/**
* First TYPE parameter value, lowercased (used as `context`).
*/
private function firstType(Property $prop): ?string {
$type = $prop['TYPE'] ?? null;
if ($type === null) {
return null;
}
$parts = $type->getParts();
return isset($parts[0]) && $parts[0] !== '' ? strtolower((string)$parts[0]) : null;
}
/**
* @param array<int,string> $parts
*/
private function part(array $parts, int $index): ?string {
if (!isset($parts[$index])) {
return null;
}
$value = trim((string)$parts[$index]);
return $value === '' ? null : $value;
}
/**
* @return list<string>
*/
private function categories(VCardComponent $vCard): array {
$tags = [];
foreach ($vCard->select('CATEGORIES') as $prop) {
foreach ($prop->getParts() as $part) {
$part = trim((string)$part);
if ($part !== '') {
$tags[] = $part;
}
}
}
return $tags;
}
/**
* Best-effort parse of a vCard date value; null if unparseable.
*/
private function date(string $value): ?\DateTimeImmutable {
$value = trim($value);
if ($value === '') {
return null;
}
// Common compact form YYYYMMDD.
if (preg_match('/^\d{8}$/', $value)) {
$date = \DateTimeImmutable::createFromFormat('!Ymd', $value);
return $date ?: null;
}
try {
return new \DateTimeImmutable($value);
} catch (Throwable) {
return null;
}
}
private function key(): string {
return bin2hex(random_bytes(8));
}
}
+9
View File
@@ -14,6 +14,15 @@ class Module extends ModuleInstanceAbstract implements ModuleBrowserInterface
public function __construct()
{ }
public function boot(): void
{
// Load module-local vendored dependencies (e.g. sabre/vobject for VCF import).
$vendorAutoload = __DIR__ . '/vendor/autoload.php';
if (file_exists($vendorAutoload)) {
require_once $vendorAutoload;
}
}
public function handle(): string
{
return 'people_manager';
+20
View File
@@ -0,0 +1,20 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: Sebastian Krupinski <krupinski01@gmail.com>
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace KTXM\PeopleManager\Stream;
/**
* Implemented by a stream event that declares, up front, how many data frames
* are expected to follow. When such an event leads a stream generator, the
* envelope writer folds its value into the `control:start` frame's `total`
* (the progress denominator) instead of emitting it as a `data` frame.
*/
interface ExpectedTotal {
public function expectedTotal(): int;
}