Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 49 additions & 8 deletions src/Migration/Destination.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@

abstract class Destination extends Target
{
/**
* @var array{rootResourceId: string, rootResourceType: string, rootResourceChildId: string}|null
*/
private ?array $resourceSelector = null;

/**
* Source
*/
Expand Down Expand Up @@ -34,14 +39,50 @@ public function run(
string $rootResourceId = '',
string $rootResourceType = '',
): void {
$this->source->run(
$resources,
function (array $resources) use ($callback) {
$this->import($resources, $callback);
},
$rootResourceId,
$rootResourceType,
);
$import = function (array $resources) use ($callback) {
$this->import($resources, $callback);
};

if ($this->resourceSelector !== null) {
$this->source->runWithResourceSelector(
$resources,
$import,
$this->resourceSelector['rootResourceId'],
$this->resourceSelector['rootResourceType'],
$this->resourceSelector['rootResourceChildId'],
);

return;
}

$this->source->run($resources, $import, $rootResourceId, $rootResourceType);
}

/**
* Transfer resources using separate, opaque root and child IDs.
*
* @param array<string> $resources Resources to transfer
* @param callable(array<Resource>): void $callback to run after transfer
*/
public function runWithResourceSelector(
array $resources,
callable $callback,
string $rootResourceId,
string $rootResourceType,
string $rootResourceChildId,
): void {
$previousResourceSelector = $this->resourceSelector;
$this->resourceSelector = [
'rootResourceId' => $rootResourceId,
'rootResourceType' => $rootResourceType,
'rootResourceChildId' => $rootResourceChildId,
];

try {
$this->run($resources, $callback, $rootResourceId, $rootResourceType);
} finally {
$this->resourceSelector = $previousResourceSelector;
}
}

/**
Expand Down
18 changes: 16 additions & 2 deletions src/Migration/Destinations/Appwrite.php
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,9 @@ class Appwrite extends Destination
*/
private array $provisioningDatabases = [];

/** Whether the destination project's database metadata supports lifecycle status. */
private ?bool $databaseStatusSupported = null;

/**
* @param string $project
* @param string $endpoint
Expand Down Expand Up @@ -241,11 +244,22 @@ private function getSupportForDatabaseStatus(): bool
// $source is a non-nullable typed property with no default; it is only set when the
// destination runs through Transfer. Guard so direct createDatabase() calls (e.g. tests)
// don't hit "must not be accessed before initialization".
if (! isset($this->source)) {
if (! isset($this->source) || ! $this->getSource()->supportsDatabaseStatus()) {
return false;
}

return $this->getSource()->supportsDatabaseStatus();
if ($this->databaseStatusSupported !== null) {
return $this->databaseStatusSupported;
}

$collection = $this->dbForProject->getCollection(self::META_DATABASES);
foreach ($collection->getAttribute('attributes', []) as $attribute) {
if ($attribute->getId() === 'status') {
return $this->databaseStatusSupported = true;
}
}

return $this->databaseStatusSupported = false;
}

/** Orphan cleanup runs only after a successful migration — a mid-run throw preserves the destination as-is. */
Expand Down
31 changes: 30 additions & 1 deletion src/Migration/Destinations/CSV.php
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ class CSV extends Destination
{
protected Device $deviceForFiles;
protected string $resourceId;
private ?string $resourceChildId = null;
protected string $directory;
protected string $outputFile;
protected Local $local;
Expand Down Expand Up @@ -54,6 +55,34 @@ public function __construct(
}
}

public static function fromResourceIds(
Device $deviceForFiles,
string $databaseId,
string $tableId,
string $directory,
string $filename,
array $allowedColumns = [],
string $delimiter = ',',
string $enclosure = '"',
string $escape = '"',
bool $includeHeaders = true,
): self {
$destination = new self(
deviceForFiles: $deviceForFiles,
resourceId: $databaseId,
directory: $directory,
filename: $filename,
allowedColumns: $allowedColumns,
delimiter: $delimiter,
enclosure: $enclosure,
escape: $escape,
includeHeaders: $includeHeaders,
);
$destination->resourceChildId = $tableId;

return $destination;
}

public static function getName(): string
{
return 'CSV';
Expand Down Expand Up @@ -193,7 +222,7 @@ public function shutdown(): void
UtopiaResource::TYPE_ROW,
Transfer::GROUP_DATABASES,
'Error cleaning up: ' . $this->local->getRoot(),
$this->resourceId
$this->resourceChildId ?? $this->resourceId
));
}
}
Expand Down
23 changes: 22 additions & 1 deletion src/Migration/Destinations/JSON.php
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ class JSON extends Destination
{
protected Device $deviceForFiles;
protected string $resourceId;
private ?string $resourceChildId = null;
protected string $directory;
protected string $outputFile;
protected Local $local;
Expand Down Expand Up @@ -56,6 +57,26 @@ public function __construct(
}
}

public static function fromResourceIds(
Device $deviceForFiles,
string $databaseId,
string $tableId,
string $directory,
string $filename,
array $allowedColumns = [],
): self {
$destination = new self(
deviceForFiles: $deviceForFiles,
resourceId: $databaseId,
directory: $directory,
filename: $filename,
allowedColumns: $allowedColumns,
);
$destination->resourceChildId = $tableId;

return $destination;
}

public static function getName(): string
{
return 'JSON';
Expand Down Expand Up @@ -217,7 +238,7 @@ public function shutdown(): void
UtopiaResource::TYPE_ROW,
Transfer::GROUP_DATABASES,
'Error cleaning up: ' . $this->local->getRoot(),
$this->resourceId
$this->resourceChildId ?? $this->resourceId
));
}
}
Expand Down
25 changes: 24 additions & 1 deletion src/Migration/Source.php
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,8 @@ public function callback(array $resources): void
*/
public function run(array $resources, callable $callback, string $rootResourceId = '', string $rootResourceType = ''): void
{
$previousRootResourceId = $this->rootResourceId;
$previousRootResourceType = $this->rootResourceType;
$this->rootResourceId = $rootResourceId;
$this->rootResourceType = $rootResourceType;

Expand All @@ -115,7 +117,28 @@ public function run(array $resources, callable $callback, string $rootResourceId
$this->cache->addAll($prunedResources);
};

$this->exportResources($resources);
try {
$this->exportResources($resources);
} finally {
$this->rootResourceId = $previousRootResourceId;
$this->rootResourceType = $previousRootResourceType;
}
}

/**
* Transfer resources using separate, opaque root and child IDs.
*
* @param array<string> $resources Resources to transfer
* @param callable $callback Callback to run after transfer
*/
public function runWithResourceSelector(
array $resources,
callable $callback,
string $rootResourceId,
string $rootResourceType,
string $rootResourceChildId,
): void {
$this->run($resources, $callback, $rootResourceId, $rootResourceType);
}

/**
Expand Down
87 changes: 64 additions & 23 deletions src/Migration/Sources/Appwrite.php
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
use Appwrite\Services\Teams;
use Appwrite\Services\Users;
use Appwrite\Services\Webhooks;
use Override;
use Utopia\Database\Database as UtopiaDatabase;
use Utopia\Database\DateTime as UtopiaDateTime;
use Utopia\Database\Document as UtopiaDocument;
Expand Down Expand Up @@ -117,6 +118,8 @@ class Appwrite extends Source

private Proxy $proxy;

private ?string $resourceChildId = null;

/**
* @var callable(UtopiaDocument $database|null): UtopiaDatabase
*/
Expand Down Expand Up @@ -180,6 +183,58 @@ public static function getName(): string
return 'Appwrite';
}

#[Override]
public function run(
array $resources,
callable $callback,
string $rootResourceId = '',
string $rootResourceType = '',
): void {
$previousResourceChildId = $this->resourceChildId;

if ($this->resourceChildId === null) {
$this->resourceChildId = '';

if (
$rootResourceId !== ''
&& \array_key_exists($rootResourceType, Resource::DATABASE_TYPE_RESOURCE_MAP)
&& \str_contains($rootResourceId, ':')
) {
[$rootResourceId, $this->resourceChildId] = \explode(':', $rootResourceId, 2);
}
}

try {
parent::run($resources, $callback, $rootResourceId, $rootResourceType);
} finally {
$this->resourceChildId = $previousResourceChildId;
}
}

#[Override]
public function runWithResourceSelector(
array $resources,
callable $callback,
string $rootResourceId,
string $rootResourceType,
string $rootResourceChildId,
): void {
$previousResourceChildId = $this->resourceChildId;
$this->resourceChildId = $rootResourceChildId;

try {
parent::runWithResourceSelector(
$resources,
$callback,
$rootResourceId,
$rootResourceType,
$rootResourceChildId,
);
} finally {
$this->resourceChildId = $previousResourceChildId;
}
}

public function getSourceType(): string
{
return $this->source;
Expand Down Expand Up @@ -1073,18 +1128,8 @@ private function exportDatabases(int $batchSize, array $resources = []): void
while (true) {
$queries = [$this->reader->queryLimit($batchSize)];

if ($this->rootResourceId !== '' && ($this->rootResourceType === Resource::TYPE_DATABASE || $this->rootResourceType === Resource::TYPE_DATABASE_DOCUMENTSDB)) {
$targetDatabaseId = $this->rootResourceId;

// Handle database:collection format - extract database ID
if (\str_contains($this->rootResourceId, ':')) {
$parts = \explode(':', $this->rootResourceId, 2);
if (\count($parts) === 2) {
$targetDatabaseId = $parts[0];
}
}

$queries[] = $this->reader->queryEqual('$id', [$targetDatabaseId]);
if ($this->rootResourceId !== '' && \array_key_exists($this->rootResourceType, Resource::DATABASE_TYPE_RESOURCE_MAP)) {
$queries[] = $this->reader->queryEqual('$id', [$this->rootResourceId]);
$queries[] = $this->reader->queryLimit(1);
}

Expand Down Expand Up @@ -1148,24 +1193,20 @@ private function exportEntities(string $databaseName, int $batchSize): void
$queries = [$this->reader->queryLimit($batchSize)];
$tables = [];

// Filter to specific table if rootResourceType is database with database:collection format
// Filter to a specific table or collection when its database root has a child set,
// or when the root itself is a table.
if (
$this->rootResourceId !== '' &&
$this->rootResourceType === Resource::TYPE_DATABASE &&
\str_contains($this->rootResourceId, ':')
$this->resourceChildId !== '' &&
$this->rootResourceType === $databaseName
) {
$parts = \explode(':', $this->rootResourceId, 2);
if (\count($parts) === 2) {
$targetTableId = $parts[1]; // table ID
$queries[] = $this->reader->queryEqual('$id', [$targetTableId]);
$queries[] = $this->reader->queryLimit(1);
}
$queries[] = $this->reader->queryEqual('$id', [$this->resourceChildId]);
$queries[] = $this->reader->queryLimit(1);
} elseif (
$this->rootResourceId !== '' &&
$this->rootResourceType === Resource::TYPE_TABLE
) {
$targetTableId = $this->rootResourceId;
$queries[] = $this->reader->queryEqual('$id', [$targetTableId]);
$queries[] = $this->reader->queryEqual('$id', [$this->rootResourceId]);
$queries[] = $this->reader->queryLimit(1);
} elseif ($lastTable) {
$queries[] = $this->reader->queryCursorAfter($lastTable);
Expand Down
Loading
Loading