Skip to content

Commit adc27ae

Browse files
author
Snehil
committed
Seo data dumping to local db
1 parent 85dde53 commit adc27ae

10 files changed

Lines changed: 1225 additions & 670 deletions

File tree

‎helpers/getSeoDatabase.js‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import path from "path";
2+
import knex from "knex";
3+
4+
const dbPath = path.join(process.cwd(), "seo/db.sqlite3");
5+
const db = knex({
6+
client: "better-sqlite3",
7+
connection: {
8+
filename: dbPath,
9+
},
10+
useNullAsDefault: true,
11+
migrations: {
12+
tableName: "knex_migrations",
13+
},
14+
});
15+
16+
export default db;

‎knexfile.js‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
const path = require('path');
2+
// Update with your config settings.
3+
4+
module.exports = {
5+
6+
development: {
7+
client: 'better-sqlite3',
8+
connection: {
9+
filename: path.join(__dirname, 'seo/db.sqlite3')
10+
},
11+
migrations: {
12+
tableName: 'knex_migrations'
13+
},
14+
useNullAsDefault: true
15+
}
16+
17+
};

‎migrations/20230330092345_json_tables.js‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/**
2+
* @param { import("knex").Knex } knex
3+
* @returns { Promise<void> }
4+
*/
5+
exports.up = function (knex) {
6+
return knex.schema
7+
.createTable("Whois", (t) => {
8+
t.string("creator");
9+
t.integer("id").primary();
10+
t.string("name");
11+
t.string("address");
12+
// ownerType: OwnerType;
13+
t.string("ownerType");
14+
})
15+
.createTable("Users", (t) => {
16+
t.string("address").primary();
17+
t.json("data");
18+
})
19+
.createTable("Daos", (t) => {
20+
t.string("address").primary();
21+
t.json("data");
22+
})
23+
.createTable("Repositories", (t) => {
24+
t.integer("id").primary();
25+
t.string("name");
26+
t.string("ownerUsername");
27+
t.string("ownerAddress");
28+
t.string("ownerType");
29+
t.timestamp("updatedAt");
30+
t.json("data");
31+
})
32+
.createTable("Issues", (t) => {
33+
t.integer('id').primary();
34+
t.integer("iid");
35+
t.integer("repositoryId");
36+
t.json("data");
37+
})
38+
.createTable("PullRequests", (t) => {
39+
t.integer('id').primary();
40+
t.integer("iid");
41+
t.integer("baseRepositoryId");
42+
t.json("data");
43+
})
44+
.createTable("Comments", (t) => {
45+
t.integer("id").primary();
46+
t.integer("repositoryId");
47+
t.integer("parentIid");
48+
t.string("parent");
49+
t.json("data");
50+
});
51+
};
52+
53+
/**
54+
* @param { import("knex").Knex } knex
55+
* @returns { Promise<void> }
56+
*/
57+
exports.down = function (knex) {
58+
return knex.schema
59+
.dropTableIfExists("Whois")
60+
.dropTableIfExists("Users")
61+
.dropTableIfExists("Daos")
62+
.dropTableIfExists("Repositories")
63+
.dropTableIfExists("Issues")
64+
.dropTableIfExists("PullRequests")
65+
.dropTableIfExists("Comments");
66+
};

‎package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"private": true,
55
"scripts": {
66
"dev": "next dev",
7-
"prebuild": "node scripts/dump-paths.mjs",
7+
"prebuild": "knex migrate:up && node scripts/dump-paths.mjs",
88
"build": "next build && next export",
99
"postbuild": "cp ./seo/sitemap.xml ./public/",
1010
"start": "next start",
@@ -27,6 +27,7 @@
2727
"@rpldy/upload-button": "0.13.5",
2828
"@rpldy/upload-drop-zone": "0.13.5",
2929
"@rpldy/uploady": "^0.13.5",
30+
"better-sqlite3": "^8.2.0",
3031
"bip39": "^3.0.3",
3132
"classnames": "^2.3.1",
3233
"crypto-js": "^4.0.0",
@@ -40,6 +41,7 @@
4041
"globby": "^11.0.1",
4142
"graphql": "^15.8.0",
4243
"graphql-ws": "^5.6.4",
44+
"knex": "^2.4.2",
4345
"lodash": "^4.17.21",
4446
"mime-types": "^2.1.35",
4547
"next": "^13.1.6",

‎pages/[userId]/[repositoryId]/index.js‎

Lines changed: 48 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -30,57 +30,70 @@ import getAllRepositoryBranch from "../../../helpers/getAllRepositoryBranch";
3030
import getAllRepositoryTag from "../../../helpers/getAllRepositoryTag";
3131
import getUser from "../../../helpers/getUser";
3232
import getDao from "../../../helpers/getDao";
33+
import validAddress from "../../../helpers/validAddress";
3334

3435
const atob = (base64) => {
3536
return Buffer.from(base64, "base64").toString("binary");
3637
};
3738

3839
export async function getStaticProps({ params }) {
3940
try {
40-
const fs = (await import("fs")).default;
41-
const repositories = JSON.parse(
42-
fs.readFileSync("./seo/dump-repositories.json")
43-
);
44-
45-
const r = find(
46-
repositories,
47-
(r) =>
48-
r.name === params.repositoryId &&
49-
(r.owner.id === params.userId || r.owner.username === params.userId)
50-
);
41+
const db = (await import("../../../helpers/getSeoDatabase")).default;
42+
let r;
43+
if (validAddress.test(params.userId)) {
44+
r = JSON.parse(
45+
(
46+
await db
47+
.first("*")
48+
.from("Repositories")
49+
.where({ name: params.repositoryId, ownerAddress: params.userId })
50+
).data
51+
);
52+
} else {
53+
r = JSON.parse(
54+
(
55+
await db
56+
.first("*")
57+
.from("Repositories")
58+
.where({ name: params.repositoryId, ownerUsername: params.userId })
59+
).data
60+
);
61+
}
5162

5263
if (r) {
53-
let branchSha = getBranchSha(r.defaultBranch, r.branches, r.tags);
54-
55-
const entitiesRes = await getContent(r.id, branchSha, null, null);
56-
57-
const commitHistory = await getCommitHistory(r.id, branchSha, null, 1);
58-
59-
const readmeRegex = new RegExp(/^README/gi);
60-
let readmeFile = null;
61-
for (let i = 0; i < entitiesRes?.content?.length; i++) {
62-
if (readmeRegex.test(entitiesRes.content[i].name)) {
63-
const readme = await getContent(
64-
r.id,
65-
branchSha,
66-
entitiesRes.content[i].name
67-
);
68-
if (readme?.content[0]) {
69-
readmeFile = atob(readme.content[0].content);
70-
}
71-
}
72-
}
64+
// let branchSha = getBranchSha(r.defaultBranch, r.branches, r.tags);
65+
66+
// const entitiesRes = await getContent(r.id, branchSha, null, null);
67+
68+
// const commitHistory = await getCommitHistory(r.id, branchSha, null, 1);
69+
70+
// const readmeRegex = new RegExp(/^README/gi);
71+
// let readmeFile = null;
72+
// for (let i = 0; i < entitiesRes?.content?.length; i++) {
73+
// if (readmeRegex.test(entitiesRes.content[i].name)) {
74+
// const readme = await getContent(
75+
// r.id,
76+
// branchSha,
77+
// entitiesRes.content[i].name
78+
// );
79+
// if (readme?.content[0]) {
80+
// readmeFile = atob(readme.content[0].content);
81+
// }
82+
// }
83+
// }
7384
return {
7485
props: {
7586
repository: r,
76-
entitiesRes,
77-
commitHistory: { commits: [{}], ...commitHistory },
78-
readmeFile,
87+
// entitiesRes,
88+
// commitHistory: { commits: [{}], ...commitHistory },
89+
// readmeFile,
7990
},
8091
revalidate: 1,
8192
};
8293
}
83-
} catch (e) {}
94+
} catch (e) {
95+
console.error(e);
96+
}
8497
return {
8598
props: {},
8699
};

‎pages/[userId]/[repositoryId]/issues/[issueIid]/index.js‎

Lines changed: 41 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -36,37 +36,59 @@ import IssueBountyView from "../../../../../components/repository/bountiesView";
3636
import Link from "next/link";
3737
import filter from "lodash/filter";
3838
import getIssueCommentAll from "../../../../../helpers/getIssueCommentAll";
39+
import validAddress from "../../../../../helpers/validAddress";
3940

4041
export async function getStaticProps({ params }) {
4142
try {
42-
const fs = (await import("fs")).default;
43-
const issues = JSON.parse(fs.readFileSync("./seo/dump-issues.json")),
44-
repositories = JSON.parse(
45-
fs.readFileSync("./seo/dump-repositories.json")
46-
),
47-
comments = JSON.parse(fs.readFileSync("./seo/dump-comments.json"));
48-
49-
const r = find(
50-
repositories,
51-
(r) =>
52-
r.name === params.repositoryId &&
53-
(r.owner.id === params.userId || r.owner.username === params.userId)
54-
);
43+
const db = (await import("../../../../../helpers/getSeoDatabase")).default;
44+
let r;
45+
if (validAddress.test(params.userId)) {
46+
r = JSON.parse(
47+
(
48+
await db
49+
.first("*")
50+
.from("Repositories")
51+
.where({ name: params.repositoryId, ownerAddress: params.userId })
52+
).data
53+
);
54+
} else {
55+
r = JSON.parse(
56+
(
57+
await db
58+
.first("*")
59+
.from("Repositories")
60+
.where({ name: params.repositoryId, ownerUsername: params.userId })
61+
).data
62+
);
63+
}
5564

5665
if (r) {
57-
const i = find(
58-
issues,
59-
(t) => t.iid === params.issueIid && t.repositoryId === r.id
66+
const i = JSON.parse(
67+
(
68+
await db
69+
.first("*")
70+
.from("Issues")
71+
.where({ iid: params.issueIid, repositoryId: r.id })
72+
).data
6073
);
6174
if (i) {
62-
const cs = filter(comments, (c) => i.comments.includes(c.id));
75+
let cs;
76+
if (i.commentsCount) {
77+
const csJsons = await db
78+
.select("*")
79+
.from("Comments")
80+
.where({repositoryId: r.id, parentIid: i.iid, parent: "COMMENT_PARENT_ISSUE"})
81+
cs = csJsons.map((j) => JSON.parse(j.data));
82+
}
6383
return {
64-
props: { repository: r, issue: i, comments: cs },
84+
props: { repository: r, issue: i, comments: cs || [] },
6585
revalidate: 1,
6686
};
6787
}
6888
}
69-
} catch (e) {}
89+
} catch (e) {
90+
console.error(e);
91+
}
7092
return { props: {} };
7193
}
7294

0 commit comments

Comments
 (0)