Skip to content

Commit a33e6a2

Browse files
CaffeineFusionCaffeineFusion
committed
merging stream processing uploads into index.js
1 parent 88a8a1e commit a33e6a2

2 files changed

Lines changed: 30 additions & 54 deletions

File tree

‎experiments/gcp/upload.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,10 @@ function uploadByURL(url, bucket, fileName) {
1515
let file = bucket.file(fileName);
1616
return new Promise((resolve, reject) => {
1717
request(url)
18-
.on('response', (response) => { response.pause(); resolve(response); });
19-
}).then((response) => { return response.pipe(file.createWriteStream({ gzip: true })); });
18+
.on('response', (response) => { response.pause(); resolve(response); })
19+
.on('error', (reject));
20+
}).then((response) => { return response.pipe(file.createWriteStream({ gzip: true })); })
21+
.catch((error) => { return { error }; }); // TODO: Add more detailed error handling. - statusCode etc.
2022
}
2123

2224
exports.upload = function (url, fileName) {

‎index.js‎

Lines changed: 26 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,31 @@
11
'use strict';
22
const url = require('url');
3-
const uuid = require('uuidv4');
3+
//const uuid = require('uuidv4');
44
const request = require('request');
5-
const rp = require('request-promise');
5+
//const rp = require('request-promise');
66

77
const config = require('./config.json');
88

9-
const Storage = require('@google-cloud/storage');
9+
const {Storage} = require('@google-cloud/storage');
1010
const storage = new Storage({
1111
projectId: config.project_id,
1212
keyFilename: config.keys.cloud_storage.path
1313
});
1414

15-
1615
function parseURL(urlString) {
1716
try {
1817
return url.parse(urlString);
1918
}
2019
catch(e) {
21-
//
20+
// TODO
2221
}
23-
console.log('index.js, parseURL not written');
24-
//return(url);
2522
}
2623

2724
function checkURLStatus(url) {
2825
return new Promise((resolve, reject) => {
29-
const request = require('request');
30-
3126
request({method: 'HEAD', url}, function (error, response) {
32-
if(error || response.statusCode !== 200) {
27+
if(error || response.statusCode !== 200)
3328
reject({url, statusCode:response.statusCode, error:error?error:'inaccessibleURL', message:'Remote URL could not be reached.'});
34-
}
3529
resolve(url);
3630
});
3731
});
@@ -52,46 +46,30 @@ function verifyURL(url) {
5246
});
5347
}
5448

55-
// For testing
56-
async function uploadLocalFile(fileName, bucketName) {
57-
await storage.bucket(bucketName).upload(fileName, {
58-
gzip: true,
59-
metadata: {
60-
cacheControl: 'public, max-age=31536000',
61-
},
62-
});
63-
}
64-
65-
/*
66-
const bucket = gcs.bucket('bucket_name');
67-
const gcsname = 'test.pdf';
68-
const file = bucket.file(gcsname);
69-
var pdfdata = "binary_pdf_file_string";
70-
var buff = Buffer.from(pdfdata, 'binary').toString('utf-8');
49+
/* // Testing purposes:
50+
async function uploadLocalFile(bucket, fileName) {
7151
72-
const stream = file.createWriteStream({
73-
metadata: {
74-
contentType: 'application/pdf'
75-
}
76-
});
77-
stream.on('error', (err) => {
78-
console.log(err);
79-
});
80-
stream.on('finish', () => {
81-
console.log(gcsname);
82-
});
83-
stream.end(new Buffer(buff, 'base64'));
52+
await bucket.upload(fileName, { gzip: true });
53+
}
8454
*/
8555

86-
function uploadByURL(url, endpoint, location) {
87-
//let message = { uploadedFiles: `${imageURL}` };
88-
let file = myBucket.file('my-file');
89-
request(url)
90-
.pipe(endpoint.upload());
56+
// Returns a Promise object which resolves to a stream.
57+
function uploadByURL(url, bucket, fileName) {
58+
let file = bucket.file(fileName);
59+
return new Promise((resolve, reject) => {
60+
request(url)
61+
.on('response', (response) => { response.pause(); resolve(response); }) // Create, then pause and return the stream
62+
.on('error', (reject));
63+
})
64+
.then((response) => { return response.pipe(file.createWriteStream({ gzip: true })); })
65+
.catch((error) => { return { error }; }); // TODO: Add more detailed error handling. - statusCode etc.
66+
}
9167

92-
//res.status(200).json(message);x
68+
function upload(url, fileName) {
69+
return uploadByURL(url, storage.bucket(config.bucket_name), fileName);
9370
}
9471

72+
9573
function getImage(req, res) {
9674
let message = req.query.message || req.body.message || 'Hello World!';
9775
switch (req.get('content-type')) {
@@ -112,7 +90,7 @@ function getImage(req, res) {
11290
// Lambdas
11391

11492
/**
115-
* images - served as our /images webhook.
93+
* images - /images webhook.
11694
* @param {[type]} req [description]
11795
* @param {[type]} res [description]
11896
* @return {[type]} [description]
@@ -122,10 +100,11 @@ exports.images = (req, res) => {
122100
let urls = res.body.urls;
123101
switch(req.method) {
124102
case 'POST':
103+
// For each URL, verify, upload it, then aggregate the results to return as JSON object.
125104
Promise.all(
126105
urls.map((url) => {
127106
return verifyURL(url)
128-
.then(uploadByURL);
107+
.then((url) => { upload(url, url); }); // naming convention will likely need to change for multi-users
129108
})
130109
)
131110
// TODO: Change status IDs depending on error/s from Promises.
@@ -137,8 +116,3 @@ exports.images = (req, res) => {
137116
break;
138117
}
139118
};
140-
141-
exports.upload = (req, res) => {
142-
143-
res.status(200).send({});
144-
};

0 commit comments

Comments
 (0)