Skip to content

Commit 915a8bf

Browse files
ChiragAgg5kChiragAgg5k
committed
Merge branch '1.9.x' into feat-rust-sdk
2 parents ce5d5cf + 1ab05ac commit 915a8bf

351 files changed

Lines changed: 26052 additions & 2330 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.env‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,23 @@ _APP_REDIS_HOST=redis
3939
_APP_REDIS_PORT=6379
4040
_APP_REDIS_PASS=
4141
_APP_REDIS_USER=
42-
COMPOSE_PROFILES=mongodb
42+
COMPOSE_PROFILES=mariadb,mongodb,postgresql
4343
_APP_DB_ADAPTER=mongodb
4444
_APP_DB_HOST=mongodb
4545
_APP_DB_PORT=27017
4646
_APP_DB_SCHEMA=appwrite
4747
_APP_DB_USER=user
4848
_APP_DB_PASS=password
4949
_APP_DB_ROOT_PASS=rootsecretpassword
50+
_APP_DB_ADAPTER_DOCUMENTSDB=mongodb
51+
_APP_DB_HOST_DOCUMENTSDB=mongodb
52+
_APP_DB_PORT_DOCUMENTSDB=27017
53+
_APP_DB_ADAPTER_VECTORSDB=postgresql
54+
_APP_DB_HOST_VECTORSDB=postgresql
55+
_APP_DB_PORT_VECTORSDB=5432
56+
_APP_EMBEDDING_MODELS=embeddinggemma
57+
_APP_EMBEDDING_ENDPOINT='http://ollama:11434/api/embed'
58+
_APP_EMBEDDING_TIMEOUT=30000
5059
_APP_STORAGE_DEVICE=Local
5160
_APP_STORAGE_S3_ACCESS_KEY=
5261
_APP_STORAGE_S3_SECRET=

‎.github/workflows/ci.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,8 @@ jobs:
394394
Webhooks,
395395
VCS,
396396
Messaging,
397-
Migrations
397+
Migrations,
398+
Project
398399
]
399400
include:
400401
- service: Databases

‎app/config/collections.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
$common = include __DIR__ . '/collections/common.php';
55
$projects = include __DIR__ . '/collections/projects.php';
66
$databases = include __DIR__ . '/collections/databases.php';
7+
$vectorsdb = include __DIR__ . '/collections/vectorsdb.php';
78
$platform = include __DIR__ . '/collections/platform.php';
89
$logs = include __DIR__ . '/collections/logs.php';
910

@@ -26,6 +27,7 @@
2627
$collections = [
2728
'buckets' => $buckets,
2829
'databases' => $databases,
30+
'vectorsdb' => $vectorsdb,
2931
'projects' => array_merge_recursive($projects, $common),
3032
'console' => array_merge_recursive($platform, $common),
3133
'logs' => $logs,

‎app/config/collections/projects.php‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,15 @@
6161
'array' => false,
6262
'filters' => [],
6363
],
64+
[
65+
'$id' => ID::custom('database'),
66+
'type' => Database::VAR_STRING,
67+
'size' => 128,
68+
'required' => false,
69+
'signed' => true,
70+
'array' => false,
71+
'filters' => [],
72+
]
6473
],
6574
'indexes' => [
6675
[

‎app/config/collections/vectorsdb.php‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
<?php
2+
3+
use Utopia\Database\Database;
4+
use Utopia\Database\Helpers\ID;
5+
6+
return [
7+
'collections' => [
8+
'$collection' => ID::custom('databases'),
9+
'$id' => ID::custom('collections'),
10+
'name' => 'Collections',
11+
'attributes' => [
12+
[
13+
'$id' => ID::custom('databaseInternalId'),
14+
'type' => Database::VAR_STRING,
15+
'format' => '',
16+
'size' => Database::LENGTH_KEY,
17+
'signed' => true,
18+
'required' => true,
19+
'default' => null,
20+
'array' => false,
21+
'filters' => [],
22+
],
23+
[
24+
'$id' => ID::custom('databaseId'),
25+
'type' => Database::VAR_STRING,
26+
'signed' => true,
27+
'size' => Database::LENGTH_KEY,
28+
'format' => '',
29+
'filters' => [],
30+
'required' => true,
31+
'default' => null,
32+
'array' => false,
33+
],
34+
[
35+
'$id' => ID::custom('name'),
36+
'type' => Database::VAR_STRING,
37+
'size' => 256,
38+
'required' => true,
39+
'signed' => true,
40+
'array' => false,
41+
'filters' => [],
42+
],
43+
[
44+
'$id' => ID::custom('dimension'),
45+
'type' => Database::VAR_INTEGER,
46+
'size' => 0,
47+
'required' => true,
48+
'signed' => false,
49+
'array' => false,
50+
'filters' => [],
51+
],
52+
[
53+
'$id' => ID::custom('enabled'),
54+
'type' => Database::VAR_BOOLEAN,
55+
'signed' => true,
56+
'size' => 0,
57+
'format' => '',
58+
'filters' => [],
59+
'required' => true,
60+
'default' => null,
61+
'array' => false,
62+
],
63+
[
64+
'$id' => ID::custom('documentSecurity'),
65+
'type' => Database::VAR_BOOLEAN,
66+
'signed' => true,
67+
'size' => 0,
68+
'format' => '',
69+
'filters' => [],
70+
'required' => true,
71+
'default' => null,
72+
'array' => false,
73+
],
74+
[
75+
'$id' => ID::custom('attributes'),
76+
'type' => Database::VAR_STRING,
77+
'size' => 1000000,
78+
'required' => false,
79+
'signed' => true,
80+
'array' => false,
81+
'filters' => ['subQueryAttributes'],
82+
],
83+
[
84+
'$id' => ID::custom('indexes'),
85+
'type' => Database::VAR_STRING,
86+
'size' => 1000000,
87+
'required' => false,
88+
'signed' => true,
89+
'array' => false,
90+
'filters' => ['subQueryIndexes'],
91+
],
92+
[
93+
'$id' => ID::custom('search'),
94+
'type' => Database::VAR_STRING,
95+
'format' => '',
96+
'size' => 16384,
97+
'signed' => true,
98+
'required' => false,
99+
'default' => null,
100+
'array' => false,
101+
'filters' => [],
102+
],
103+
],
104+
'defaultAttributes' => [
105+
[
106+
'$id' => ID::custom('embeddings'),
107+
'type' => Database::VAR_VECTOR,
108+
'required' => true,
109+
'signed' => false,
110+
'array' => false,
111+
'filters' => [],
112+
],
113+
[
114+
'$id' => ID::custom('metadata'),
115+
'type' => Database::VAR_OBJECT,
116+
'default' => [],
117+
'required' => false,
118+
'size' => 0,
119+
'signed' => false,
120+
'array' => false,
121+
'filters' => [],
122+
],
123+
],
124+
'indexes' => [
125+
[
126+
'$id' => ID::custom('_fulltext_search'),
127+
'type' => Database::INDEX_FULLTEXT,
128+
'attributes' => ['search'],
129+
'lengths' => [],
130+
'orders' => [],
131+
],
132+
[
133+
'$id' => ID::custom('_key_name'),
134+
'type' => Database::INDEX_KEY,
135+
'attributes' => ['name'],
136+
'lengths' => [256],
137+
'orders' => [Database::ORDER_ASC],
138+
],
139+
[
140+
'$id' => ID::custom('_key_enabled'),
141+
'type' => Database::INDEX_KEY,
142+
'attributes' => ['enabled'],
143+
'lengths' => [],
144+
'orders' => [Database::ORDER_ASC],
145+
],
146+
[
147+
'$id' => ID::custom('_key_documentSecurity'),
148+
'type' => Database::INDEX_KEY,
149+
'attributes' => ['documentSecurity'],
150+
'lengths' => [],
151+
'orders' => [Database::ORDER_ASC],
152+
],
153+
],
154+
'defaultIndexes' => [
155+
// not creating default indexes on the embeddings as it depends on the type of query users using the most
156+
[
157+
'$id' => ID::custom('_key_metadata'),
158+
'type' => Database::INDEX_OBJECT,
159+
'attributes' => ['metadata'],
160+
'lengths' => [],
161+
'orders' => [],
162+
],
163+
]
164+
]
165+
];

‎app/config/errors.php‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1206,6 +1206,11 @@
12061206
'description' => 'Migration is already in progress. You can check the status of the migration in your Appwrite Console\'s "Settings" > "Migrations".',
12071207
'code' => 409,
12081208
],
1209+
Exception::MIGRATION_DATABASE_TYPE_UNSUPPORTED => [
1210+
'name' => Exception::MIGRATION_DATABASE_TYPE_UNSUPPORTED,
1211+
'description' => 'The specified database type is not supported for CSV import or export operations.',
1212+
'code' => 400,
1213+
],
12091214

12101215
/** Realtime */
12111216
Exception::REALTIME_MESSAGE_FORMAT_INVALID => [

‎app/config/roles.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@
6262
'devKeys.write',
6363
'webhooks.read',
6464
'webhooks.write',
65+
'project.read',
66+
'project.write',
6567
'locale.read',
6668
'avatars.read',
6769
'health.read',

‎app/config/scopes/organization.php‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,4 @@
3131
"description" =>
3232
"Access to create, update, and delete project\'s development keys",
3333
],
34-
"webhooks.read" => [
35-
"description" =>
36-
"Access to read project\'s webhooks",
37-
],
38-
"webhooks.write" => [
39-
"description" =>
40-
"Access to create, update, and delete project\'s webhooks",
41-
],
4234
];

‎app/config/scopes/project.php‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,4 +180,12 @@
180180
"description" =>
181181
"Access to create, update, and delete project\'s webhooks",
182182
],
183+
"project.read" => [
184+
"description" =>
185+
"Access to read project\'s information",
186+
],
187+
"project.write" => [
188+
"description" =>
189+
"Access to update project\'s information",
190+
],
183191
];

‎app/controllers/api/migrations.php‎

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,16 @@
4343

4444
include_once __DIR__ . '/../shared/api.php';
4545

46+
function getDatabaseTransferResourceServices(string $databaseType)
47+
{
48+
return match($databaseType) {
49+
DATABASE_TYPE_LEGACY,
50+
DATABASE_TYPE_TABLESDB => Transfer::GROUP_DATABASES_TABLES_DB,
51+
DATABASE_TYPE_VECTORSDB => Transfer::GROUP_DATABASES_VECTOR_DB,
52+
DATABASE_TYPE_DOCUMENTSDB => Transfer::GROUP_DATABASES_DOCUMENTS_DB
53+
};
54+
}
55+
4656
Http::post('/v1/migrations/appwrite')
4757
->groups(['api', 'migrations'])
4858
->desc('Create Appwrite migration')
@@ -427,8 +437,16 @@
427437
throw new \Exception('Unable to copy file');
428438
}
429439

440+
// getting databasetype
441+
$resources = explode(':', $resourceId);
442+
$databaseId = $resources[0];
443+
$database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
444+
$databaseType = $database->getAttribute('type');
445+
if (!in_array($databaseType, CSV_ALLOWED_DATABASE_TYPES)) {
446+
throw new Exception(Exception::MIGRATION_DATABASE_TYPE_UNSUPPORTED, 'Database type not supported for csv');
447+
}
430448
$fileSize = $deviceForMigrations->getFileSize($newPath);
431-
$resources = Transfer::extractServices([Transfer::GROUP_DATABASES]);
449+
$resources = Transfer::extractServices([getDatabaseTransferResourceServices($databaseType)]);
432450

433451
$migration = $dbForProject->createDocument('migrations', new Document([
434452
'$id' => $migrationId,
@@ -557,13 +575,23 @@
557575
throw new Exception(Exception::GENERAL_QUERY_INVALID, $validator->getDescription());
558576
}
559577

578+
// getting databasetype
579+
$resources = explode(':', $resourceId);
580+
$databaseId = $resources[0];
581+
$database = $authorization->skip(fn () => $dbForProject->getDocument('databases', $databaseId));
582+
$databaseType = $database->getAttribute('type');
583+
if (!in_array($databaseType, CSV_ALLOWED_DATABASE_TYPES)) {
584+
throw new Exception(Exception::MIGRATION_DATABASE_TYPE_UNSUPPORTED, 'Database type not supported for csv');
585+
}
586+
$resources = Transfer::extractServices([getDatabaseTransferResourceServices($databaseType)]);
587+
560588
$migration = $dbForProject->createDocument('migrations', new Document([
561589
'$id' => ID::unique(),
562590
'status' => 'pending',
563591
'stage' => 'init',
564592
'source' => Appwrite::getName(),
565593
'destination' => CSV::getName(),
566-
'resources' => Transfer::extractServices([Transfer::GROUP_DATABASES]),
594+
'resources' => $resources,
567595
'resourceId' => $resourceId,
568596
'resourceType' => Resource::TYPE_DATABASE,
569597
'statusCounters' => '{}',
@@ -713,12 +741,11 @@
713741
->param('projectID', '', new Text(512), "Source's Project ID")
714742
->param('key', '', new Text(512), "Source's API Key")
715743
->inject('response')
716-
->inject('dbForProject')
717-
->inject('project')
718-
->inject('user')
719-
->action(function (array $resources, string $endpoint, string $projectID, string $key, Response $response) {
744+
->inject('getDatabasesDB')
745+
->action(function (array $resources, string $endpoint, string $projectID, string $key, Response $response, callable $getDatabasesDB) {
746+
720747
try {
721-
$appwrite = new Appwrite($projectID, $endpoint, $key);
748+
$appwrite = new Appwrite($projectID, $endpoint, $key, $getDatabasesDB);
722749
$report = $appwrite->report($resources);
723750
} catch (\Throwable $e) {
724751
throw new Exception(

0 commit comments

Comments
 (0)