Skip to content

Commit 4393328

Browse files
fazafaza
committed
Progress bar and other optimizations
1 parent 9e8db61 commit 4393328

File tree

5 files changed

+290
-175
lines changed

5 files changed

+290
-175
lines changed

‎bin/helper.js‎

Lines changed: 50 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,18 @@ import DGitHelper from "./lib/dgit.js";
1111
import LineHelper from "./lib/line.js";
1212
import Arweave from "arweave";
1313
import {
14-
getRefsOnArweave,
1514
pushGitObject,
1615
makeDataItem,
17-
updateRef,
16+
makeUpdateRefDataItem,
1817
parseArgitRemoteURI,
1918
postBundledTransaction,
2019
} from "./lib/arweave.js";
20+
import { getAllRefs } from "./lib/graphql.js";
2121

2222
import * as deepHash from "arweave/node/lib/deepHash.js";
2323
import ArweaveData from "arweave-data/pkg/dist-node/index.js";
24+
import pkg from "cli-progress";
25+
const { SingleBar, Presets } = pkg;
2426

2527
const _timeout = async (duration) => {
2628
return new Promise((resolve, reject) => {
@@ -199,10 +201,10 @@ export default class Helper {
199201
const spinner = ora("Fetching remote refs from arweave").start();
200202

201203
try {
202-
const refs = await getRefsOnArweave(this._arweave, this.url);
204+
const refs = await getAllRefs(this._arweave, this.url);
203205

204206
spinner.succeed("Remote refs fetched from arweave");
205-
return Object.fromEntries(refs);
207+
return refs;
206208
} catch (err) {
207209
spinner.fail("Failed to fetch remote refs from arweave");
208210
throw err;
@@ -246,7 +248,6 @@ export default class Helper {
246248

247249
return (async (resolve, reject) => {
248250
let spinner;
249-
let head;
250251
let txHash;
251252
let mapping = {};
252253
const puts = [];
@@ -255,7 +256,6 @@ export default class Helper {
255256
try {
256257
const refs = await this._fetchRefs();
257258
const remote = refs[dst];
258-
const dataItems = [];
259259

260260
const srcBranch = src.split("/").pop();
261261
const dstBranch = dst.split("/").pop();
@@ -292,27 +292,45 @@ export default class Helper {
292292
this._die();
293293
}
294294

295+
// update ref
296+
puts.push(
297+
makeUpdateRefDataItem(
298+
this.ArData,
299+
this.wallet,
300+
this.url,
301+
dst,
302+
objects[0]
303+
)
304+
);
305+
295306
// collect git objects
296-
try {
297-
spinner = ora(
298-
"Collecting git objects [this may take a while]"
299-
).start();
300-
301-
for (const oid of objects) {
302-
const object = await this.git.load(oid);
303-
304-
const dataItem = await makeDataItem(
305-
this.ArData,
306-
this.wallet,
307-
this.url,
308-
oid,
309-
object
310-
);
311-
console.error("oid", oid);
312-
dataItems.push(dataItem);
313-
}
307+
spinner = ora("Collecting git objects [this may take a while]").start();
308+
309+
const bar1 = new SingleBar(
310+
{ stream: process.stderr },
311+
Presets.shades_classic
312+
);
313+
314+
bar1.start(objects.length, 0);
314315

315-
head = objects[0];
316+
try {
317+
objects.map((oid) =>
318+
puts.push(
319+
(async (bar) => {
320+
bar.increment();
321+
322+
const object = await this.git.load(oid);
323+
const dataItem = await makeDataItem(
324+
this.ArData,
325+
this.wallet,
326+
this.url,
327+
oid,
328+
object
329+
);
330+
return dataItem;
331+
})(bar1)
332+
)
333+
);
316334

317335
spinner.succeed("Git objects collected");
318336
} catch (err) {
@@ -323,10 +341,13 @@ export default class Helper {
323341
// upload git objects
324342
try {
325343
spinner = ora("Uploading git objects to arweave").start();
344+
const dataItems = await Promise.all(puts);
345+
bar1.stop();
326346
await postBundledTransaction(
327347
this._arweave,
328348
this.ArData,
329349
this.wallet,
350+
this.url,
330351
dataItems
331352
);
332353
spinner.succeed("Git objects uploaded to arweave");
@@ -339,11 +360,12 @@ export default class Helper {
339360

340361
// register on chain
341362
try {
342-
spinner = ora(`Updating ref ${dst} ${head} on arweave`).start();
343-
await updateRef(this._arweave, this.wallet, this.url, dst, head);
363+
spinner = ora(`Updating ref ${dst} ${objects[0]} on arweave`).start();
364+
spinner.succeed(`Updated ref ${dst} ${objects[0]} successfully`);
344365
} catch (err) {
345366
spinner.fail(
346-
`Failed to update ref ${dst} ${head} on arweave: ` + err.message
367+
`Failed to update ref ${dst} ${objects[0]} on arweave: ` +
368+
err.message
347369
);
348370
this._die();
349371
}

‎bin/lib/arweave.js‎

Lines changed: 33 additions & 146 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import * as smartweave from "smartweave";
2-
import * as utils from "arweave/node/lib/utils.js";
32

43
// prettier-ignore
54
const argitRemoteURIRegex = '^dgit:\/\/([a-zA-Z0-9-_]{43})\/([A-Za-z0-9_.-]*)'
@@ -43,54 +42,29 @@ function addTransactionTags(tx, repo, txType) {
4342
return tx;
4443
}
4544

46-
export async function updateRef(arweave, wallet, remoteURI, name, ref) {
45+
export async function makeUpdateRefDataItem(
46+
arData,
47+
wallet,
48+
remoteURI,
49+
name,
50+
ref
51+
) {
4752
const { repoName } = parseArgitRemoteURI(remoteURI);
48-
let tx = await arweave.createTransaction({ data: ref }, wallet);
49-
tx = addTransactionTags(tx, repoName, "update-ref");
50-
tx.addTag("ref", name);
51-
tx.addTag("Content-Type", "text/plain");
52-
53-
await arweave.transactions.sign(tx, wallet); // Sign transaction
54-
return arweave.transactions.post(tx); // Post transaction
55-
}
56-
57-
export async function getRef(arweave, remoteURI, name) {
58-
const query = {
59-
op: "and",
60-
expr1: repoQuery(remoteURI),
61-
expr2: {
62-
op: "and",
63-
expr1: { op: "equals", expr1: "Type", expr2: "update-ref" },
64-
expr2: { op: "equals", expr1: "ref", expr2: name },
53+
const tags = [
54+
{ name: "App-Name", value: "dgit" },
55+
{ name: "version", value: "0.0.1" },
56+
{ name: "Repo", value: repoName },
57+
{ name: "Type", value: "update-ref" },
58+
{
59+
name: "Unix-Time",
60+
value: Math.round(new Date().getTime() / 1000).toString(),
6561
},
66-
};
67-
const txids = await arweave.arql(query);
68-
const tx_rows = await Promise.all(
69-
txids.map(async (txid) => {
70-
let tx_row = {};
71-
const tx = await arweave.transactions.get(txid);
72-
tx.get("tags").forEach((tag) => {
73-
const key = tag.get("name", { decode: true, string: true });
74-
const value = tag.get("value", { decode: true, string: true });
75-
if (key === "Unix-Time") tx_row.unixTime = value;
76-
});
77-
78-
tx_row.oid = await arweave.transactions.getData(txid, {
79-
decode: true,
80-
string: true,
81-
});
82-
83-
return tx_row;
84-
})
85-
);
86-
87-
if (tx_rows.length === 0) return "0000000000000000000000000000000000000000";
62+
{ name: "ref", value: name },
63+
{ name: "Content-Type", value: "text/plain" },
64+
];
8865

89-
// descending order
90-
tx_rows.sort((a, b) => {
91-
Number(b.unixTime) - Number(a.unixTime);
92-
});
93-
return tx_rows[0].oid;
66+
const item = await arData.createData({ data: ref, tags }, wallet);
67+
return await arData.sign(item, wallet);
9468
}
9569

9670
export async function pushGitObject(arweave, wallet, remoteURI, oid, object) {
@@ -120,7 +94,6 @@ export const makeDataItem = async (
12094
objectBuf
12195
) => {
12296
const { repoName } = parseArgitRemoteURI(remoteURI);
123-
const data = utils.default.bufferTob64Url(objectBuf);
12497
const tags = [
12598
{ name: "App-Name", value: "dgit" },
12699
{ name: "version", value: "0.0.1" },
@@ -134,16 +107,18 @@ export const makeDataItem = async (
134107
{ name: "Content-Type", value: "application/octet-stream" },
135108
];
136109

137-
const item = await arData.createData({ data, tags }, wallet);
110+
const item = await arData.createData({ data: objectBuf, tags }, wallet);
138111
return await arData.sign(item, wallet);
139112
};
140113

141114
export const postBundledTransaction = async (
142115
arweave,
143116
arData,
144117
wallet,
118+
remoteURI,
145119
dataItems
146120
) => {
121+
const { repoName } = parseArgitRemoteURI(remoteURI);
147122
const bundle = await arData.bundleData(dataItems);
148123
const data = JSON.stringify(bundle);
149124
const tx = await arweave.createTransaction({ data }, wallet);
@@ -152,6 +127,7 @@ export const postBundledTransaction = async (
152127
tx.addTag("Content-Type", "application/json");
153128
tx.addTag("App-Name", "dgit");
154129
tx.addTag("Type", "git-objects-bundle");
130+
tx.addTag("Repo", repoName);
155131

156132
await arweave.transactions.sign(tx, wallet);
157133
const uploader = await arweave.transactions.getUploader(tx);
@@ -162,56 +138,6 @@ export const postBundledTransaction = async (
162138
`${uploader.pctComplete}% complete, ${uploader.uploadedChunks}/${uploader.totalChunks}`
163139
);
164140
}
165-
};
166-
167-
export async function fetchGitObjects(arweave, remoteURI) {
168-
const query = {
169-
op: "and",
170-
expr1: repoQuery(remoteURI),
171-
expr2: { op: "equals", expr1: "Type", expr2: "push-git-object" },
172-
};
173-
const txids = await arweave.arql(query);
174-
const objects = await Promise.all(
175-
txids.map(async (txid) => {
176-
const tx = await arweave.transactions.get(txid);
177-
let oid = "";
178-
tx.get("tags").forEach((tag) => {
179-
const key = tag.get("name", { decode: true, string: true });
180-
const value = tag.get("value", { decode: true, string: true });
181-
if (key === "oid") oid = value;
182-
});
183-
const data = await arweave.transactions.getData(txid, { decode: true });
184-
return { data, oid };
185-
})
186-
);
187-
return objects;
188-
}
189-
190-
export async function pushPackfile(
191-
arweave,
192-
wallet,
193-
remoteURI,
194-
oldoid,
195-
oid,
196-
packfile
197-
) {
198-
const { repoName } = parseArgitRemoteURI(remoteURI);
199-
200-
let tx = await arweave.createTransaction({ data: packfile.packfile }, wallet);
201-
tx = addTransactionTags(tx, repoName, "send-pack");
202-
tx.addTag("oid", oid);
203-
tx.addTag("oldoid", oldoid);
204-
tx.addTag("filename", packfile.filename);
205-
206-
await arweave.transactions.sign(tx, wallet);
207-
let uploader = await arweave.transactions.getUploader(tx);
208-
209-
while (!uploader.isComplete) {
210-
await uploader.uploadChunk();
211-
console.log(
212-
`${uploader.pctComplete}% complete, ${uploader.uploadedChunks}/${uploader.totalChunks}`
213-
);
214-
}
215141

216142
// Send fee to PST holders
217143
const contractState = await smartweave.readContract(arweave, contractId);
@@ -223,70 +149,31 @@ export async function pushPackfile(
223149
);
224150
pstTx.addTag("App-Name", "dgit");
225151
pstTx.addTag("version", "0.0.1");
152+
pstTx.addTag("Repo", repoName);
226153

227154
await arweave.transactions.sign(pstTx, wallet);
228155
await arweave.transactions.post(pstTx);
229-
}
156+
};
230157

231-
export async function fetchPackfiles(arweave, remoteURI) {
158+
export async function fetchGitObjects(arweave, arData, remoteURI) {
232159
const query = {
233160
op: "and",
234161
expr1: repoQuery(remoteURI),
235-
expr2: { op: "equals", expr1: "Type", expr2: "send-pack" },
162+
expr2: { op: "equals", expr1: "Type", expr2: "push-git-object" },
236163
};
237164
const txids = await arweave.arql(query);
238-
const packfiles = await Promise.all(
165+
const objects = await Promise.all(
239166
txids.map(async (txid) => {
240167
const tx = await arweave.transactions.get(txid);
241-
let filename = "";
168+
let oid = "";
242169
tx.get("tags").forEach((tag) => {
243170
const key = tag.get("name", { decode: true, string: true });
244171
const value = tag.get("value", { decode: true, string: true });
245-
if (key === "filename") filename = value;
172+
if (key === "oid") oid = value;
246173
});
247174
const data = await arweave.transactions.getData(txid, { decode: true });
248-
return { data, filename };
249-
})
250-
);
251-
return packfiles;
252-
}
253-
254-
export async function getRefsOnArweave(arweave, remoteURI) {
255-
const refs = new Map();
256-
const query = {
257-
op: "and",
258-
expr1: repoQuery(remoteURI),
259-
expr2: { op: "equals", expr1: "Type", expr2: "update-ref" },
260-
};
261-
const txids = await arweave.arql(query);
262-
const tx_rows = await Promise.all(
263-
txids.map(async (txid) => {
264-
let ref = {};
265-
const tx = await arweave.transactions.get(txid);
266-
tx.get("tags").forEach((tag) => {
267-
const key = tag.get("name", { decode: true, string: true });
268-
const value = tag.get("value", { decode: true, string: true });
269-
if (key === "Unix-Time") ref.unixTime = value;
270-
else if (key === "ref") ref.name = value;
271-
});
272-
273-
ref.oid = await arweave.transactions.getData(txid, {
274-
decode: true,
275-
string: true,
276-
});
277-
278-
return ref;
175+
return { data, oid };
279176
})
280177
);
281-
282-
// descending order
283-
tx_rows.sort((a, b) => {
284-
Number(b.unixTime) - Number(a.unixTime);
285-
});
286-
287-
tx_rows.forEach((ref) => {
288-
if (!refs.has(ref.name)) refs.set(ref.name, ref.oid);
289-
});
290-
291-
return refs;
178+
return objects;
292179
}

0 commit comments

Comments
 (0)