Skip to content

Commit 553f5d2

Browse files
msukkaribrendan-kellam
msukkari
and
brendan-kellam
authored
Add tenant ID concept into web app and backend (#160)
* hacked together a example of using zoekt grpc api * provide tenant id to zoekt git indexer * update zoekt version to point to multitenant branch * pipe tenant id through header to zoekt * remove incorrect submodule reference and settings typo * update zoekt commit * remove unused yarn script * remove unused grpc client in web server * remove unneeded deps and improve tenant id log * pass tenant id when creating repo in db * add mt yarn script * add nocheckin comment to tenant id in v2 schema --------- Co-authored-by: bkellam <[email protected]>
1 parent 3c3140e commit 553f5d2

12 files changed

Lines changed: 73 additions & 4 deletions

File tree

‎package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,9 @@
77
"build": "yarn workspaces run build",
88
"test": "yarn workspaces run test",
99
"dev": "npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web",
10-
"dev:zoekt": "export PATH=\"$PWD/bin:$PATH\" && zoekt-webserver -index .sourcebot/index -rpc",
10+
"dev:mt": "npm-run-all --print-label --parallel dev:zoekt:mt dev:backend dev:web",
11+
"dev:zoekt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=none && zoekt-webserver -index .sourcebot/index -rpc",
12+
"dev:zoekt:mt": "export PATH=\"$PWD/bin:$PATH\" && export SRC_TENANT_ENFORCEMENT_MODE=strict && zoekt-webserver -index .sourcebot/index -rpc",
1113
"dev:backend": "yarn workspace @sourcebot/backend dev:watch",
1214
"dev:web": "yarn workspace @sourcebot/web dev"
1315
},

‎packages/backend/src/config.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ export const syncConfig = async (configPath: string, db: PrismaClient, signal: A
3535
const gitHubRepos = await getGitHubReposFromConfig(repoConfig, signal, ctx);
3636
const hostUrl = repoConfig.url ?? 'https://github.com';
3737
const hostname = repoConfig.url ? new URL(repoConfig.url).hostname : 'github.com';
38+
const tenantId = repoConfig.tenantId ?? 0;
3839

3940
await Promise.all(gitHubRepos.map((repo) => {
4041
const repoName = `${hostname}/${repo.full_name}`;
@@ -51,6 +52,7 @@ export const syncConfig = async (configPath: string, db: PrismaClient, signal: A
5152
name: repoName,
5253
isFork: repo.fork,
5354
isArchived: !!repo.archived,
55+
tenantId: tenantId,
5456
metadata: {
5557
'zoekt.web-url-type': 'github',
5658
'zoekt.web-url': repo.html_url,
@@ -101,6 +103,7 @@ export const syncConfig = async (configPath: string, db: PrismaClient, signal: A
101103
external_codeHostUrl: hostUrl,
102104
cloneUrl: cloneUrl.toString(),
103105
name: repoName,
106+
tenantId: 0, // TODO: add support for tenantId in GitLab config
104107
isFork,
105108
isArchived: project.archived,
106109
metadata: {

‎packages/backend/src/schemas/v2.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,10 @@ export interface GitHubConfig {
7272
* @minItems 1
7373
*/
7474
topics?: string[];
75+
/**
76+
* @nocheckin
77+
*/
78+
tenantId?: number;
7579
exclude?: {
7680
/**
7781
* Exclude forked repositories from syncing.

‎packages/backend/src/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ interface BaseRepository {
1313
codeHost?: string;
1414
topics?: string[];
1515
sizeInBytes?: number;
16+
tenantId?: number;
1617
}
1718

1819
/**

‎packages/backend/src/zoekt.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,12 @@ export const indexGitRepository = async (repo: Repo, ctx: AppContext) => {
1010
const revisions = [
1111
'HEAD'
1212
];
13+
14+
const tenantId = repo.tenantId ?? 0;
15+
const shardPrefix = `${tenantId}_${repo.id}`;
1316

1417
const repoPath = getRepoPath(repo, ctx);
15-
const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -shard_prefix ${repo.id} ${repoPath}`;
18+
const command = `zoekt-git-index -allow_missing_branches -index ${ctx.indexPath} -file_limit ${DEFAULT_SETTINGS.maxFileSize} -branches ${revisions.join(',')} -tenant_id ${tenantId} -shard_prefix ${shardPrefix} ${repoPath}`;
1619

1720
return new Promise<{ stdout: string, stderr: string }>((resolve, reject) => {
1821
exec(command, (error, stdout, stderr) => {

‎packages/db/prisma/migrations/20250114222109_add_tenant_id_to_repo/migration.sql‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
Warnings:
3+
4+
- Added the required column `tenantId` to the `Repo` table without a default value. This is not possible if the table is not empty.
5+
6+
*/
7+
-- RedefineTables
8+
PRAGMA defer_foreign_keys=ON;
9+
PRAGMA foreign_keys=OFF;
10+
CREATE TABLE "new_Repo" (
11+
"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
12+
"name" TEXT NOT NULL,
13+
"createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
14+
"updatedAt" DATETIME NOT NULL,
15+
"indexedAt" DATETIME,
16+
"isFork" BOOLEAN NOT NULL,
17+
"isArchived" BOOLEAN NOT NULL,
18+
"metadata" JSONB NOT NULL,
19+
"cloneUrl" TEXT NOT NULL,
20+
"tenantId" INTEGER NOT NULL,
21+
"external_id" TEXT NOT NULL,
22+
"external_codeHostType" TEXT NOT NULL,
23+
"external_codeHostUrl" TEXT NOT NULL
24+
);
25+
INSERT INTO "new_Repo" ("cloneUrl", "createdAt", "external_codeHostType", "external_codeHostUrl", "external_id", "id", "indexedAt", "isArchived", "isFork", "metadata", "name", "updatedAt") SELECT "cloneUrl", "createdAt", "external_codeHostType", "external_codeHostUrl", "external_id", "id", "indexedAt", "isArchived", "isFork", "metadata", "name", "updatedAt" FROM "Repo";
26+
DROP TABLE "Repo";
27+
ALTER TABLE "new_Repo" RENAME TO "Repo";
28+
CREATE UNIQUE INDEX "Repo_external_id_external_codeHostUrl_key" ON "Repo"("external_id", "external_codeHostUrl");
29+
PRAGMA foreign_keys=ON;
30+
PRAGMA defer_foreign_keys=OFF;

‎packages/db/prisma/schema.prisma‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ model Repo {
2020
isArchived Boolean
2121
metadata Json
2222
cloneUrl String
23+
tenantId Int
2324
2425
// The id of the repo in the external service
2526
external_id String

‎packages/web/src/app/api/(server)/search/route.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,21 @@ import { NextRequest } from "next/server";
88

99
export const POST = async (request: NextRequest) => {
1010
const body = await request.json();
11-
const parsed = await searchRequestSchema.safeParseAsync(body);
11+
const tenantId = await request.headers.get("X-Tenant-ID");
12+
13+
console.log(`Search request received. Tenant ID: ${tenantId}`);
14+
15+
const parsed = await searchRequestSchema.safeParseAsync({
16+
...body,
17+
...(tenantId && { tenantId: parseInt(tenantId) }),
18+
});
1219
if (!parsed.success) {
1320
return serviceErrorResponse(
1421
schemaValidationError(parsed.error)
1522
);
1623
}
1724

25+
1826
const response = await search(parsed.data);
1927
if (isServiceError(response)) {
2028
return serviceErrorResponse(response);

‎packages/web/src/lib/schemas.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ export const searchRequestSchema = z.object({
44
query: z.string(),
55
maxMatchDisplayCount: z.number(),
66
whole: z.boolean().optional(),
7+
tenantId: z.number().optional(),
78
});
89

910

‎packages/web/src/lib/server/searchService.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ const aliasPrefixMappings: Record<string, zoektPrefixes> = {
3434
"revision:": zoektPrefixes.branch,
3535
}
3636

37-
export const search = async ({ query, maxMatchDisplayCount, whole }: SearchRequest): Promise<SearchResponse | ServiceError> => {
37+
export const search = async ({ query, maxMatchDisplayCount, whole, tenantId }: SearchRequest): Promise<SearchResponse | ServiceError> => {
3838
// Replace any alias prefixes with their corresponding zoekt prefixes.
3939
for (const [prefix, zoektPrefix] of Object.entries(aliasPrefixMappings)) {
4040
query = query.replaceAll(prefix, zoektPrefix);
@@ -53,9 +53,17 @@ export const search = async ({ query, maxMatchDisplayCount, whole }: SearchReque
5353
}
5454
});
5555

56+
let header: Record<string, string> = {};
57+
if (tenantId) {
58+
header = {
59+
"X-Tenant-ID": tenantId.toString()
60+
};
61+
}
62+
5663
const searchResponse = await zoektFetch({
5764
path: "/api/search",
5865
body,
66+
header,
5967
method: "POST",
6068
});
6169

0 commit comments

Comments
 (0)