Skip to content

Commit 466acf9

Browse files
hTraphTrap
authored
Merge pull request #4 from gitopia/optimize-branch
Optimize branch upload
2 parents 8297298 + b07d17a commit 466acf9

8 files changed

Lines changed: 191 additions & 116 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# git-remote-gitopia
22

3+
[![](https://img.shields.io/npm/v/@gitopia/git-remote-gitopia?color=)](https://www.npmjs.com/package/@gitopia/git-remote-gitopia)
34
git remote helper for [Gitopia](https://gitopia.org)
45
PST Fee of 0.01 AR is applicable on git push.
56

‎bin/helper.js‎

Lines changed: 42 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
import debug from "debug";
22
import fs from "fs-extra";
3-
import json from "jsonfile";
4-
import Level from "level";
53
import ora from "ora";
6-
import os from "os";
74
import path from "path";
85
import shell from "shelljs";
96
import GitHelper from "./lib/git.js";
@@ -17,13 +14,12 @@ import {
1714
postBundledTransaction,
1815
} from "./lib/arweave.js";
1916
import { getAllRefs } from "./lib/graphql.js";
17+
import { newProgressBar } from "./lib/util.js";
2018

2119
import * as deepHash from "arweave/node/lib/deepHash.js";
2220
import ArweaveBundles from "arweave-bundles";
23-
import pkg from "cli-progress";
24-
const { SingleBar, Presets } = pkg;
2521

26-
export const VERSION = "0.1.6"
22+
export const VERSION = "0.1.7";
2723

2824
const _timeout = async (duration) => {
2925
return new Promise((resolve, reject) => {
@@ -262,25 +258,32 @@ export default class Helper {
262258

263259
try {
264260
const refs = await this._fetchRefs();
265-
const remote = refs[dst];
266261

267262
const srcBranch = src.split("/").pop();
268-
const dstBranch = dst.split("/").pop();
263+
const srcOid = shell
264+
.exec(`git rev-parse ${srcBranch}`, {
265+
silent: true,
266+
})
267+
.stdout.split("\n")[0];
268+
269+
let remoteBranches = "";
270+
for (const [ref, oid] of Object.entries(refs)) {
271+
const isValidLocalRef =
272+
shell.exec(`git cat-file -p ${oid}`, { silent: true }).code === 0;
273+
274+
if (isValidLocalRef) {
275+
remoteBranches = remoteBranches.concat("^", oid, " ");
276+
}
277+
}
269278

270-
const revListCmd = remote
271-
? `git rev-list --objects --left-only ${srcBranch}...${this.name}/${dstBranch}`
272-
: `git rev-list --objects ${srcBranch}`;
279+
const revListCmd = `git rev-list --objects ${srcBranch} ${remoteBranches}`;
273280

274281
const objects = shell
275282
.exec(revListCmd, { silent: true })
276283
.stdout.split("\n")
277284
.slice(0, -1)
278285
.map((object) => object.substr(0, 40));
279286

280-
if (objects.length === 0) {
281-
this._exit()
282-
}
283-
284287
// checking permissions
285288
try {
286289
spinner = ora(`Checking permissions over ${this.address}`).start();
@@ -305,63 +308,49 @@ export default class Helper {
305308

306309
// update ref
307310
puts.push(
308-
makeUpdateRefDataItem(
309-
this.ArData,
310-
this.wallet,
311-
this.url,
312-
dst,
313-
objects[0]
314-
)
311+
makeUpdateRefDataItem(this.ArData, this.wallet, this.url, dst, srcOid)
315312
);
316313

317-
// collect git objects
318-
spinner = ora("Collecting git objects [this may take a while]").start();
314+
const bar1 = newProgressBar();
319315

320-
const bar1 = new SingleBar(
321-
{ stream: process.stderr },
322-
Presets.shades_classic
323-
);
316+
// collect git objects
317+
console.error("Collecting git objects [this may take a while]");
324318

325319
bar1.start(objects.length, 0);
326320

327321
try {
328-
objects.map((oid) =>
322+
for (const oid of objects) {
323+
const object = await this.git.load(oid);
324+
bar1.increment();
325+
329326
puts.push(
330-
(async (bar) => {
331-
bar.increment();
332-
333-
const object = await this.git.load(oid);
334-
const dataItem = await makeDataItem(
335-
this.ArData,
336-
this.wallet,
337-
this.url,
338-
oid,
339-
object
340-
);
341-
return dataItem;
342-
})(bar1)
343-
)
344-
);
327+
makeDataItem(this.ArData, this.wallet, this.url, oid, object)
328+
);
329+
}
345330

346-
spinner.succeed("Git objects collected");
331+
// spinner.succeed("Git objects collected");
347332
} catch (err) {
348-
spinner.fail("Failed to collect git objects: " + err.message);
333+
// spinner.fail("Failed to collect git objects: " + err.message);
349334
this._die();
350335
}
351336

337+
const dataItems = await Promise.all(puts);
338+
bar1.stop();
339+
console.error("Git objects collected successfully");
340+
352341
// upload git objects
353342
try {
354-
spinner = ora("Uploading git objects to Gitopia").start();
355-
const dataItems = await Promise.all(puts);
356-
bar1.stop();
343+
console.error(
344+
"Uploading git objects to Gitopia [this may take a while]"
345+
);
357346
await postBundledTransaction(
358347
this._arweave,
359348
this.ArData,
360349
this.wallet,
361350
this.url,
362351
dataItems
363352
);
364-
spinner.succeed("Git objects uploaded to Gitopia");
353+
console.error("Git objects uploaded to Gitopia successfully");
365354
} catch (err) {
366355
spinner.fail(
367356
"Failed to upload git objects to Gitopia: " + err.message
@@ -371,12 +360,11 @@ export default class Helper {
371360

372361
// register on chain
373362
try {
374-
spinner = ora(`Updating ref ${dst} ${objects[0]} on Gitopia`).start();
375-
spinner.succeed(`Updated ref ${dst} ${objects[0]} successfully`);
363+
spinner = ora(`Updating ref ${dst} ${srcOid} on Gitopia`).start();
364+
spinner.succeed(`Updated ref ${dst} ${srcOid} successfully`);
376365
} catch (err) {
377366
spinner.fail(
378-
`Failed to update ref ${dst} ${objects[0]} on Gitopia: ` +
379-
err.message
367+
`Failed to update ref ${dst} ${srcOid} on Gitopia: ` + err.message
380368
);
381369
this._die();
382370
}

‎bin/lib/arweave.js‎

Lines changed: 48 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
11
import * as smartweave from "smartweave";
2-
import shell from "shelljs"
3-
import { VERSION } from "../helper.js"
2+
import shell from "shelljs";
3+
import pkg from "bignumber.js";
4+
const { BigNumber } = pkg;
5+
6+
import { VERSION } from "../helper.js";
7+
import { newProgressBar } from "./util.js";
48

59
// prettier-ignore
610
const argitRemoteURIRegex = '^gitopia:\/\/([a-zA-Z0-9-_]{43})\/([A-Za-z0-9_.-]*)'
7-
const contractId = "UvjBvJUy8pOMR_lf85tBDaJD0jF85G5Ayj1p2h7yols";
11+
const contractId = "1ljLAR55OhtenU0iDWkLGT6jF4ApxeQd5P0gXNyNJXg";
812

913
export function parseArgitRemoteURI(remoteURI) {
1014
const matchGroups = remoteURI.match(argitRemoteURIRegex);
@@ -27,19 +31,21 @@ export async function makeUpdateRefDataItem(
2731
{ name: "Version", value: "0.0.2" },
2832
{ name: "Ref", value: ref },
2933
{ name: "Type", value: "update-ref" },
30-
{ name: "App-Name", value: "gitopia" },
34+
{ name: "App-Name", value: "Gitopia" },
3135
{
3236
name: "Unix-Time",
3337
value: Math.round(new Date().getTime() / 1000).toString(),
3438
},
35-
{ name: "Content-Type", value: "text/plain" },
39+
{ name: "Content-Type", value: "application/json" },
3640
];
3741

38-
const numCommits = shell.exec(`git rev-list --count ${ref}`, { silent: true }).stdout.trim();
42+
const numCommits = shell
43+
.exec(`git rev-list --count ${ref}`, { silent: true })
44+
.stdout.trim();
3945
const obj = {
4046
oid,
41-
numCommits
42-
}
47+
numCommits,
48+
};
4349
const data = JSON.stringify(obj);
4450

4551
const item = await arData.createData({ data, tags }, wallet);
@@ -59,7 +65,7 @@ export const makeDataItem = async (
5965
{ name: "Version", value: "0.0.2" },
6066
{ name: "Repo", value: repoName },
6167
{ name: "Type", value: "git-object" },
62-
{ name: "App-Name", value: "gitopia" },
68+
{ name: "App-Name", value: "Gitopia" },
6369
{
6470
name: "Unix-Time",
6571
value: Math.round(new Date().getTime() / 1000).toString(),
@@ -83,43 +89,63 @@ export const postBundledTransaction = async (
8389
const data = JSON.stringify(bundle);
8490
const tx = await arweave.createTransaction({ data }, wallet);
8591
tx.addTag("Repo", repoName);
86-
tx.addTag("Version", "0.0.2")
92+
tx.addTag("Version", "0.0.2");
8793
tx.addTag("Type", "git-objects-bundle");
88-
tx.addTag("App-Name", "gitopia");
94+
tx.addTag("App-Name", "Gitopia");
8995
tx.addTag("Bundle-Format", "json");
9096
tx.addTag("Bundle-Version", "1.0.0");
9197
tx.addTag("Content-Type", "application/json");
9298
tx.addTag("Helper", VERSION);
99+
tx.addTag("Unix-Time", Math.round(new Date().getTime() / 1000).toString());
93100

94101
// Push triggered from gitopia mirror action
95102
if (process.env.GITHUB_SHA) {
96-
tx.addTag("Origin", "gitopia-mirror-action")
103+
tx.addTag("Origin", "gitopia-mirror-action");
97104
} else {
98-
tx.addTag("Origin", "git-remote-gitopia")
105+
tx.addTag("Origin", "git-remote-gitopia");
99106
}
100107

101108
await arweave.transactions.sign(tx, wallet);
102109
const uploader = await arweave.transactions.getUploader(tx);
103110

111+
const bar = newProgressBar();
112+
bar.start(uploader.totalChunks, 0);
113+
104114
while (!uploader.isComplete) {
105115
await uploader.uploadChunk();
106-
console.error(
107-
`${uploader.pctComplete}% complete, ${uploader.uploadedChunks}/${uploader.totalChunks}`
108-
);
116+
bar.update(uploader.uploadedChunks);
109117
}
110118

119+
bar.stop();
120+
111121
// Send fee to PST holders
112-
const contractState = await smartweave.default.readContract(arweave, contractId);
113-
const holder = smartweave.default.selectWeightedPstHolder(contractState.balances);
114-
// send a fee. You should inform the user about this fee and amount.
122+
const contractState = await smartweave.default.readContract(
123+
arweave,
124+
contractId
125+
);
126+
const holder = smartweave.default.selectWeightedPstHolder(
127+
contractState.balances
128+
);
129+
130+
// PST Fee
131+
const txFee = new BigNumber(tx.reward);
132+
const pstFee = txFee.multipliedBy(0.1);
133+
134+
const quantity = pstFee.isGreaterThan(
135+
BigNumber(arweave.ar.arToWinston("0.01"))
136+
)
137+
? pstFee.toFixed(0)
138+
: arweave.ar.arToWinston("0.01");
139+
115140
const pstTx = await arweave.createTransaction(
116-
{ target: holder, quantity: arweave.ar.arToWinston("0.01") },
141+
{ target: holder, quantity },
117142
wallet
118143
);
119-
pstTx.addTag("Bundle-TxID", tx.id)
144+
pstTx.addTag("Bundle-Txid", tx.id);
120145
pstTx.addTag("Repo", repoName);
121146
pstTx.addTag("Version", "0.0.2");
122-
pstTx.addTag("App-Name", "gitopia");
147+
pstTx.addTag("App-Name", "Gitopia");
148+
pstTx.addTag("Unix-Time", Math.round(new Date().getTime() / 1000).toString());
123149

124150
await arweave.transactions.sign(pstTx, wallet);
125151
await arweave.transactions.post(pstTx);

‎bin/lib/git.js‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@ import gitP from "simple-git/promise.js";
77
import pkg from "smart-buffer";
88
const { SmartBuffer } = pkg;
99
import zlib from "zlib";
10-
import pkg1 from "cli-progress";
11-
const { SingleBar, Presets } = pkg1;
1210

1311
import { fetchGitObjects } from "./graphql.js";
1412

@@ -42,20 +40,11 @@ export default class GitHelper {
4240
this.helper.url
4341
);
4442

45-
const bar1 = new SingleBar(
46-
{ stream: process.stderr },
47-
Presets.shades_classic
48-
);
49-
50-
bar1.start(objects.length, 0);
51-
5243
for (const object of objects) {
53-
bar1.increment();
5444
dumps.push(this.dump(object.oid, object.data));
5545
}
5646

5747
await Promise.all(dumps);
58-
bar1.stop();
5948
}
6049

6150
/***** fs-related methods *****/

0 commit comments

Comments
 (0)