Skip to content

Commit 846d73b

Browse files
brendan-kellambrendan-kellam
authored
Connection management (#183)
1 parent afff36f commit 846d73b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+2758
-371
lines changed

‎packages/backend/src/connectionManager.ts‎

Lines changed: 60 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -80,54 +80,68 @@ export class ConnectionManager implements IConnectionManager {
8080
const abortController = new AbortController();
8181

8282
type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;
83-
const repoData: RepoData[] = await (async () => {
84-
switch (config.type) {
85-
case 'github': {
86-
const gitHubRepos = await getGitHubReposFromConfig(config, orgId, this.db, abortController.signal);
87-
const hostUrl = config.url ?? 'https://github.com';
88-
const hostname = config.url ? new URL(config.url).hostname : 'github.com';
89-
90-
return gitHubRepos.map((repo) => {
91-
const repoName = `${hostname}/${repo.full_name}`;
92-
const cloneUrl = new URL(repo.clone_url!);
93-
94-
const record: RepoData = {
95-
external_id: repo.id.toString(),
96-
external_codeHostType: 'github',
97-
external_codeHostUrl: hostUrl,
98-
cloneUrl: cloneUrl.toString(),
99-
name: repoName,
100-
isFork: repo.fork,
101-
isArchived: !!repo.archived,
102-
org: {
103-
connect: {
104-
id: orgId,
83+
const repoData: RepoData[] = (
84+
await (async () => {
85+
switch (config.type) {
86+
case 'github': {
87+
const gitHubRepos = await getGitHubReposFromConfig(config, orgId, this.db, abortController.signal);
88+
const hostUrl = config.url ?? 'https://github.com';
89+
const hostname = config.url ? new URL(config.url).hostname : 'github.com';
90+
91+
return gitHubRepos.map((repo) => {
92+
const repoName = `${hostname}/${repo.full_name}`;
93+
const cloneUrl = new URL(repo.clone_url!);
94+
95+
const record: RepoData = {
96+
external_id: repo.id.toString(),
97+
external_codeHostType: 'github',
98+
external_codeHostUrl: hostUrl,
99+
cloneUrl: cloneUrl.toString(),
100+
imageUrl: repo.owner.avatar_url,
101+
name: repoName,
102+
isFork: repo.fork,
103+
isArchived: !!repo.archived,
104+
org: {
105+
connect: {
106+
id: orgId,
107+
},
105108
},
106-
},
107-
connections: {
108-
create: {
109-
connectionId: job.data.connectionId,
110-
}
111-
},
112-
metadata: {
113-
'zoekt.web-url-type': 'github',
114-
'zoekt.web-url': repo.html_url,
115-
'zoekt.name': repoName,
116-
'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
117-
'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
118-
'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
119-
'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
120-
'zoekt.archived': marshalBool(repo.archived),
121-
'zoekt.fork': marshalBool(repo.fork),
122-
'zoekt.public': marshalBool(repo.private === false)
123-
},
124-
};
125-
126-
return record;
127-
})
109+
connections: {
110+
create: {
111+
connectionId: job.data.connectionId,
112+
}
113+
},
114+
metadata: {
115+
'zoekt.web-url-type': 'github',
116+
'zoekt.web-url': repo.html_url,
117+
'zoekt.name': repoName,
118+
'zoekt.github-stars': (repo.stargazers_count ?? 0).toString(),
119+
'zoekt.github-watchers': (repo.watchers_count ?? 0).toString(),
120+
'zoekt.github-subscribers': (repo.subscribers_count ?? 0).toString(),
121+
'zoekt.github-forks': (repo.forks_count ?? 0).toString(),
122+
'zoekt.archived': marshalBool(repo.archived),
123+
'zoekt.fork': marshalBool(repo.fork),
124+
'zoekt.public': marshalBool(repo.private === false)
125+
},
126+
};
127+
128+
return record;
129+
})
130+
}
131+
case 'gitlab': {
132+
// @todo
133+
return [];
134+
}
128135
}
129-
}
130-
})();
136+
})()
137+
)
138+
// Filter out any duplicates by external_id and external_codeHostUrl.
139+
.filter((repo, index, self) => {
140+
return index === self.findIndex(r =>
141+
r.external_id === repo.external_id &&
142+
r.external_codeHostUrl === repo.external_codeHostUrl
143+
);
144+
})
131145

132146
// @note: to handle orphaned Repos we delete all RepoToConnection records for this connection,
133147
// and then recreate them when we upsert the repos. For example, if a repo is no-longer

‎packages/backend/src/github.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ export type OctokitRepository = {
2424
topics?: string[],
2525
// @note: this is expressed in kilobytes.
2626
size?: number,
27+
owner: {
28+
avatar_url: string,
29+
}
2730
}
2831

2932
export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {

‎packages/db/prisma/migrations/20250124230248_add_name_to_connection/migration.sql‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
/*
2+
Warnings:
3+
4+
- Added the required column `name` to the `Connection` table without a default value. This is not possible if the table is not empty.
5+
6+
*/
7+
-- AlterTable
8+
ALTER TABLE "Connection" ADD COLUMN "name" TEXT NOT NULL;

‎packages/db/prisma/migrations/20250124231224_add_connection_type/migration.sql‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
/*
2+
Warnings:
3+
4+
- Added the required column `connectionType` to the `Connection` table without a default value. This is not possible if the table is not empty.
5+
6+
*/
7+
-- AlterTable
8+
ALTER TABLE "Connection" ADD COLUMN "connectionType" TEXT NOT NULL;

‎packages/db/prisma/migrations/20250203215003_add_repo_image/migration.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
-- AlterTable
2+
ALTER TABLE "Repo" ADD COLUMN "imageUrl" TEXT;

‎packages/db/prisma/schema.prisma‎

Lines changed: 28 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -27,17 +27,17 @@ enum ConnectionSyncStatus {
2727
}
2828

2929
model Repo {
30-
id Int @id @default(autoincrement())
31-
name String
32-
createdAt DateTime @default(now())
33-
updatedAt DateTime @updatedAt
34-
indexedAt DateTime?
35-
isFork Boolean
36-
isArchived Boolean
37-
metadata Json
38-
cloneUrl String
39-
connections RepoToConnection[]
40-
30+
id Int @id @default(autoincrement())
31+
name String
32+
createdAt DateTime @default(now())
33+
updatedAt DateTime @updatedAt
34+
indexedAt DateTime?
35+
isFork Boolean
36+
isArchived Boolean
37+
metadata Json
38+
cloneUrl String
39+
connections RepoToConnection[]
40+
imageUrl String?
4141
repoIndexingStatus RepoIndexingStatus @default(NEW)
4242
4343
// The id of the repo in the external service
@@ -54,15 +54,18 @@ model Repo {
5454
}
5555

5656
model Connection {
57-
id Int @id @default(autoincrement())
58-
config Json
59-
createdAt DateTime @default(now())
60-
updatedAt DateTime @updatedAt
61-
syncedAt DateTime?
62-
repos RepoToConnection[]
63-
57+
id Int @id @default(autoincrement())
58+
name String
59+
config Json
60+
createdAt DateTime @default(now())
61+
updatedAt DateTime @updatedAt
62+
syncedAt DateTime?
63+
repos RepoToConnection[]
6464
syncStatus ConnectionSyncStatus @default(SYNC_NEEDED)
6565
66+
// The type of connection (e.g., github, gitlab, etc.)
67+
connectionType String
68+
6669
// The organization that owns this connection
6770
org Org @relation(fields: [orgId], references: [id], onDelete: Cascade)
6871
orgId Int
@@ -71,10 +74,10 @@ model Connection {
7174
model RepoToConnection {
7275
addedAt DateTime @default(now())
7376
74-
connection Connection @relation(fields: [connectionId], references: [id], onDelete: Cascade)
77+
connection Connection @relation(fields: [connectionId], references: [id], onDelete: Cascade)
7578
connectionId Int
7679
77-
repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade)
80+
repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade)
7881
repoId Int
7982
8083
@@id([connectionId, repoId])
@@ -113,12 +116,12 @@ model UserToOrg {
113116
}
114117

115118
model Secret {
116-
orgId Int
117-
key String
118-
encryptedValue String
119-
iv String
119+
orgId Int
120+
key String
121+
encryptedValue String
122+
iv String
120123
121-
createdAt DateTime @default(now())
124+
createdAt DateTime @default(now())
122125
123126
org Org @relation(fields: [orgId], references: [id], onDelete: Cascade)
124127

‎packages/schemas/src/v3/connection.schema.ts‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,144 @@ const schema = {
214214
"type"
215215
],
216216
"additionalProperties": false
217+
},
218+
{
219+
"$schema": "http://json-schema.org/draft-07/schema#",
220+
"type": "object",
221+
"title": "GitLabConnectionConfig",
222+
"properties": {
223+
"type": {
224+
"const": "gitlab",
225+
"description": "GitLab Configuration"
226+
},
227+
"token": {
228+
"$ref": "#/oneOf/0/properties/token",
229+
"description": "An authentication token.",
230+
"examples": [
231+
"secret-token",
232+
{
233+
"env": "ENV_VAR_CONTAINING_TOKEN"
234+
}
235+
]
236+
},
237+
"url": {
238+
"type": "string",
239+
"format": "url",
240+
"default": "https://gitlab.com",
241+
"description": "The URL of the GitLab host. Defaults to https://gitlab.com",
242+
"examples": [
243+
"https://gitlab.com",
244+
"https://gitlab.example.com"
245+
],
246+
"pattern": "^https?:\\/\\/[^\\s/$.?#].[^\\s]*$"
247+
},
248+
"all": {
249+
"type": "boolean",
250+
"default": false,
251+
"description": "Sync all projects visible to the provided `token` (if any) in the GitLab instance. This option is ignored if `url` is either unset or set to https://gitlab.com ."
252+
},
253+
"users": {
254+
"type": "array",
255+
"items": {
256+
"type": "string"
257+
},
258+
"description": "List of users to sync with. All projects owned by the user and visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property."
259+
},
260+
"groups": {
261+
"type": "array",
262+
"items": {
263+
"type": "string"
264+
},
265+
"examples": [
266+
[
267+
"my-group"
268+
],
269+
[
270+
"my-group/sub-group-a",
271+
"my-group/sub-group-b"
272+
]
273+
],
274+
"description": "List of groups to sync with. All projects in the group (and recursive subgroups) visible to the provided `token` (if any) will be synced, unless explicitly defined in the `exclude` property. Subgroups can be specified by providing the path to the subgroup (e.g. `my-group/sub-group-a`)."
275+
},
276+
"projects": {
277+
"type": "array",
278+
"items": {
279+
"type": "string"
280+
},
281+
"examples": [
282+
[
283+
"my-group/my-project"
284+
],
285+
[
286+
"my-group/my-sub-group/my-project"
287+
]
288+
],
289+
"description": "List of individual projects to sync with. The project's namespace must be specified. See: https://docs.gitlab.com/ee/user/namespace/"
290+
},
291+
"topics": {
292+
"type": "array",
293+
"items": {
294+
"type": "string"
295+
},
296+
"minItems": 1,
297+
"description": "List of project topics to include when syncing. Only projects that match at least one of the provided `topics` will be synced. If not specified, all projects will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.",
298+
"examples": [
299+
[
300+
"docs",
301+
"core"
302+
]
303+
]
304+
},
305+
"exclude": {
306+
"type": "object",
307+
"properties": {
308+
"forks": {
309+
"type": "boolean",
310+
"default": false,
311+
"description": "Exclude forked projects from syncing."
312+
},
313+
"archived": {
314+
"type": "boolean",
315+
"default": false,
316+
"description": "Exclude archived projects from syncing."
317+
},
318+
"projects": {
319+
"type": "array",
320+
"items": {
321+
"type": "string"
322+
},
323+
"default": [],
324+
"examples": [
325+
[
326+
"my-group/my-project"
327+
]
328+
],
329+
"description": "List of projects to exclude from syncing. Glob patterns are supported. The project's namespace must be specified, see: https://docs.gitlab.com/ee/user/namespace/"
330+
},
331+
"topics": {
332+
"type": "array",
333+
"items": {
334+
"type": "string"
335+
},
336+
"description": "List of project topics to exclude when syncing. Projects that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported.",
337+
"examples": [
338+
[
339+
"tests",
340+
"ci"
341+
]
342+
]
343+
}
344+
},
345+
"additionalProperties": false
346+
},
347+
"revisions": {
348+
"$ref": "#/oneOf/0/properties/revisions"
349+
}
350+
},
351+
"required": [
352+
"type"
353+
],
354+
"additionalProperties": false
217355
}
218356
]
219357
} as const;

0 commit comments

Comments
 (0)