updated setup instructions for GCP, started experimenting with bluemix uploads

CaffeineFusion · CaffeineFusion · commit 100607a8eba9 · 2019-01-23T10:50:33.000+11:00
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,7 @@
 samples
 node_modules
 keys
+src/gcp/keys
+src/bluemix/keys
+src/azure/keys
+src/aws/keys
diff --git a/README.MD b/README.MD
@@ -49,3 +49,4 @@ AWS - *Future*
 - Subscription based upload workers
 - Swap from proprietary libraries to base http
 - Leverage relevant CDNs to improve download performance
+- Parallel multi-part uploads (https://www.npmjs.com/package/s3)
diff --git a/src/bluemix/config.json b/src/bluemix/config.json
@@ -0,0 +1,13 @@
+{
+	"project_id":"307972859230",
+	"bucket_name":"image_api_307972859230",
+	"keys": {
+		"path":"./keys",
+		"cloud_storage": {
+			"path":"./../../keys/cloudStorage.json"
+		},
+		"cloud_functions": {
+			"path":"./../../keys/functions.json"
+		}
+	}
+}
diff --git a/src/bluemix/experiments/upload.js b/src/bluemix/experiments/upload.js
@@ -13,25 +13,22 @@ let storage = new Storage.S3(config);
 
 // Returns a Promise object which resolves to a stream.
 function uploadByURL(url, bucket, fileName) {
-	return storage.putObject({
-
-	}).promise();
-
-	let file = bucket.file(fileName);
+	//let file = bucket.file(fileName);
 	return new Promise((resolve, reject) => {
 		request(url)
 			.on('response', (response) => { response.pause(); resolve(response); })
 			.on('error', (reject));
-		//.on('finish', ());
-	}).then((response) => { return response.pipe(file.createWriteStream({ gzip: true })); }) // TODO: Restructure.
+	}).then((response) => {
+		return storage.putObject({ Bucket:bucket, Key:fileName, Body:response })
+			.promise();
+	}) 
 		.catch((error) => { return { error }; }); // TODO: Add more detailed error handling. - statusCode etc.
 }
-// TODO: file.createResumableUpload
 
 exports.upload = function (url) {
 	return uploadByURL(url, storage.bucket(config.bucket_name), path.basename(url));
 };
-
+/*
 function createTextFile(bucketName, itemName, fileText) {
     console.log(`Creating new item: ${itemName}`);
     return cos.putObject({
@@ -46,6 +43,7 @@ function createTextFile(bucketName, itemName, fileText) {
         console.error(`ERROR: ${e.code} - ${e.message}\n`);
     });
 }
+*/
 
 /**
  *
diff --git a/src/gcp/README.MD b/src/gcp/README.MD
@@ -19,10 +19,13 @@ The base use case is the ability to upload remote images into a cloud storage bu
 - Resilience []
 
 ## Setup
-1. Create Project.
-2. Add Project_ID to config.json
+1. Create Project and add Project_ID to config.json   
+2. Create a service account with access to upload Google Functions and save the key in keys/gcp_functions.json   
+3. Create Bucket in Google Cloud Storage and update bucket_name in config.json  
+4. Create a service account with write permissions to your google storage bucket and save the key in keys/gcp_storage.json  
 TODO: add script to deploy.sh to add Project_ID to bucket name
 
+
 ### Service Accounts
 
 ## Leads
diff --git a/src/gcp/config.json b/src/gcp/config.json
@@ -4,10 +4,10 @@
 	"keys": {
 		"path":"./keys",
 		"cloud_storage": {
-			"path":"./keys/cloudStorage.json"
+			"path":"./keys/gcp_storage.json"
 		},
 		"cloud_functions": {
-			"path":"./keys/functions.json"
+			"path":"./keys/gcp_functions.json"
 		}
 	}
 }

Original file line number	Diff line number	Diff line change
`@@ -4,10 +4,10 @@`
`4`	`4`	`"keys": {`
`5`	`5`	`"path":"./keys",`
`6`	`6`	`"cloud_storage": {`
`7`		`- "path":"./keys/cloudStorage.json"`
	`7`	`+ "path":"./keys/gcp_storage.json"`
`8`	`8`	`},`
`9`	`9`	`"cloud_functions": {`
`10`		`- "path":"./keys/functions.json"`
	`10`	`+ "path":"./keys/gcp_functions.json"`
`11`	`11`	`}`
`12`	`12`	`}`
`13`	`13`	`}`