Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 21 additions & 4 deletions src/Migration/Destinations/Appwrite.php
Original file line number Diff line number Diff line change
Expand Up @@ -1067,10 +1067,27 @@ protected function createRecord(Row $resource, bool $isLast): bool
}
}
}
$dbForDatabases->skipRelationshipsExistCheck(fn () => $dbForDatabases->createDocuments(
'database_' . $databaseInternalId . '_collection_' . $tableInternalId,
$this->rowBuffer
));
$collectionId = 'database_' . $databaseInternalId . '_collection_' . $tableInternalId;

try {
$dbForDatabases->skipRelationshipsExistCheck(fn () => $dbForDatabases->createDocuments(
$collectionId,
$this->rowBuffer
));
} catch (DuplicateException) {
// Batch insert failed due to a duplicate document.
// Fall back to inserting one-by-one, skipping duplicates.
foreach ($this->rowBuffer as $row) {
try {
$dbForDatabases->skipRelationshipsExistCheck(fn () => $dbForDatabases->createDocument(
$collectionId,
$row
));
} catch (DuplicateException) {
// Document already exists, skip it
}
}
Comment on lines +1072 to +1089
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Fallback duplicates are reported as success, not skipped.

This loop only has buffered UtopiaDocuments, so when a duplicate is skipped here there is no way to update the matching Row resource back to STATUS_SKIPPED like the explicit duplicate branch at Lines 989-994. In practice, the migration cache/result will report these rows as successfully imported even though they were only ignored as pre-existing.

💡 Sketch of a fix
-    /**
-     * `@var` array<UtopiaDocument>
-     */
-    private array $rowBuffer = [];
+    /**
+     * `@var` array<array{resource: Row, document: UtopiaDocument}>
+     */
+    private array $rowBuffer = [];

-        $this->rowBuffer[] = new UtopiaDocument(\array_merge([
-            '$id' => $resource->getId(),
-            '$permissions' => $resource->getPermissions(),
-        ], $data));
+        $this->rowBuffer[] = [
+            'resource' => $resource,
+            'document' => new UtopiaDocument(\array_merge([
+                '$id' => $resource->getId(),
+                '$permissions' => $resource->getPermissions(),
+            ], $data)),
+        ];

-                    $dbForDatabases->skipRelationshipsExistCheck(fn () => $dbForDatabases->createDocuments(
-                        $collectionId,
-                        $this->rowBuffer
-                    ));
+                    $dbForDatabases->skipRelationshipsExistCheck(fn () => $dbForDatabases->createDocuments(
+                        $collectionId,
+                        \array_column($this->rowBuffer, 'document')
+                    ));

-                    foreach ($this->rowBuffer as $row) {
+                    foreach ($this->rowBuffer as $entry) {
                         try {
                             $dbForDatabases->skipRelationshipsExistCheck(fn () => $dbForDatabases->createDocument(
                                 $collectionId,
-                                $row
+                                $entry['document']
                             ));
                         } catch (DuplicateException) {
-                            // Document already exists, skip it
+                            $entry['resource']->setStatus(
+                                Resource::STATUS_SKIPPED,
+                                'Row has already been created'
+                            );
+                            $this->cache->update($entry['resource']);
                         }
                     }
🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@src/Migration/Destinations/Appwrite.php` around lines 1072 - 1089, The
per-row DuplicateException in the fallback loop over $this->rowBuffer currently
just swallows duplicates, causing the migration to report them as successful;
modify the inner catch (DuplicateException) block inside the
skipRelationshipsExistCheck(fn () => $dbForDatabases->createDocument(...)) loop
to mark the corresponding Row resource as STATUS_SKIPPED using the same update
routine/logic used in the explicit duplicate branch (i.e., the code that updates
the Row status to STATUS_SKIPPED when a duplicate is detected earlier), so the
migration cache/results accurately reflect skipped duplicates.

}

} finally {
$this->rowBuffer = [];
Expand Down
149 changes: 149 additions & 0 deletions tests/Migration/Unit/Destinations/AppwriteTest.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
<?php

namespace Utopia\Tests\Unit\Destinations;

use PHPUnit\Framework\TestCase;
use Utopia\Database\Database as UtopiaDatabase;
use Utopia\Database\Document as UtopiaDocument;
use Utopia\Database\Exception\Duplicate as DuplicateException;
use Utopia\Migration\Cache;
use Utopia\Migration\Destinations\Appwrite;
use Utopia\Migration\Resources\Database\Database;
use Utopia\Migration\Resources\Database\Row;
use Utopia\Migration\Resources\Database\Table;

class AppwriteTest extends TestCase
{
private function createAppwriteDestination(
UtopiaDatabase $dbForProject,
UtopiaDatabase $dbForDatabases,
): Appwrite {
$getDatabasesDB = function () use ($dbForDatabases) {
return $dbForDatabases;
};

$appwrite = new Appwrite(
'test-project',
'https://localhost',
'test-key',
$dbForProject,
$getDatabasesDB,
[]
);

$cache = new Cache();
$appwrite->registerCache($cache);

return $appwrite;
}

private function createMockDatabases(): array
{
$dbForProject = $this->createMock(UtopiaDatabase::class);

$dbDoc = new UtopiaDocument([
'$id' => 'db1',
'$sequence' => '1',
]);
$tableDoc = new UtopiaDocument([
'$id' => 'table1',
'$sequence' => '2',
'attributes' => [],
]);

$dbForProject->method('getDocument')
->willReturnCallback(function (string $collection) use ($dbDoc, $tableDoc) {
if ($collection === 'databases') {
return $dbDoc;
}
return $tableDoc;
});

$dbForDatabases = $this->createMock(UtopiaDatabase::class);

$adapter = $this->createMock(\Utopia\Database\Adapter::class);
$adapter->method('getSupportForAttributes')->willReturn(false);
$dbForDatabases->method('getAdapter')->willReturn($adapter);

$dbForDatabases->method('skipRelationshipsExistCheck')
->willReturnCallback(function (callable $callback) {
return $callback();
});

return [$dbForProject, $dbForDatabases];
}

/**
* Test that createRecord handles DuplicateException from batch createDocuments
* by falling back to one-by-one insertion and skipping duplicates.
*
* This reproduces the "Document already exists" error from Sentry (CLOUD-3JMT).
*/
public function testCreateRecordHandlesDuplicateDocuments(): void
{
[$dbForProject, $dbForDatabases] = $this->createMockDatabases();

// Batch createDocuments throws DuplicateException
$dbForDatabases->method('createDocuments')
->willThrowException(new DuplicateException('Document already exists'));

// Fallback createDocument: first succeeds, second throws duplicate (skipped)
$createDocumentCallCount = 0;
$dbForDatabases->method('createDocument')
->willReturnCallback(function (string $collection, UtopiaDocument $doc) use (&$createDocumentCallCount) {
$createDocumentCallCount++;
if ($createDocumentCallCount === 2) {
throw new DuplicateException('Document already exists');
}
return $doc;
});

$appwrite = $this->createAppwriteDestination($dbForProject, $dbForDatabases);

$database = new Database('db1', 'Test DB');
$table = new Table($database, 'Test Table', 'table1');

$method = new \ReflectionMethod(Appwrite::class, 'createRecord');

$row1 = new Row('row1', $table, ['field1' => 'value1']);
$row2 = new Row('row2', $table, ['field1' => 'value2']);

// Buffer row1 (not last)
$result1 = $method->invoke($appwrite, $row1, false);
$this->assertTrue($result1);

// Buffer row2 and flush (isLast=true) - should NOT throw
$result2 = $method->invoke($appwrite, $row2, true);
$this->assertTrue($result2);

// Verify fallback was used
$this->assertEquals(2, $createDocumentCallCount);
}

/**
* Test that when batch createDocuments succeeds, no fallback is needed.
*/
public function testCreateRecordBatchSucceeds(): void
{
[$dbForProject, $dbForDatabases] = $this->createMockDatabases();

// Batch insert succeeds
$dbForDatabases->method('createDocuments')
->willReturn(0);

// createDocument (singular) should NOT be called
$dbForDatabases->expects($this->never())->method('createDocument');

$appwrite = $this->createAppwriteDestination($dbForProject, $dbForDatabases);

$database = new Database('db1', 'Test DB');
$table = new Table($database, 'Test Table', 'table1');

$method = new \ReflectionMethod(Appwrite::class, 'createRecord');

$row = new Row('row1', $table, ['field1' => 'value1']);
$result = $method->invoke($appwrite, $row, true);

$this->assertTrue($result);
}
}
Loading