updating GCP doco

CaffeineFusion · CaffeineFusion · commit f9404ed4a237 · 2019-01-23T12:35:03.000+11:00
diff --git a/src/bluemix/experiments/get.js b/src/bluemix/experiments/get.js
@@ -21,7 +21,7 @@ let storage = new Storage.S3(s3Config);
 function getURL(fileName, bucketName) {
 	let params = { Bucket: bucketName, Key: fileName, Expires: 600 };
 	return new Promise((resolve, reject) => {
-		storage.getSignedUrl('getObject', params, function(err, url) {
+		storage.getSignedUrl('getObject', params, function(err, url) { // Doesn't work with SigV4 :(
 			if (err) reject(err);
 			else resolve(url);
 		});
diff --git a/src/bluemix/experiments/presign.js b/src/bluemix/experiments/presign.js
@@ -2,6 +2,9 @@
 
 // Initial template modifier from https://console.bluemix.net/docs/services/cloud-object-storage/hmac/presigned-urls.html#create-a-presigned-url
 
+// I've been searching around for a server-side presigning service similar to GCP,
+// at the moment it appears you need to use this client side code base, or an older version of signing (ie. not SigV4)
+
 const crypto = require('crypto');
 const moment = require('moment');
 const https = require('https');
@@ -12,48 +15,50 @@ const key = require('./../keys/ibm_storage.json');
 const accessKey = key.cos_hmac_keys.access_key_id;
 const secretKey = key.cos_hmac_keys.secret_access_key;
 const httpMethod = 'GET';
-const host = '{endpoint}';
-const region = '';
+const host = 's3.au-syd.cloud-object-storage.appdomain.cloud';
+const region = 'au-syd';
 const endpoint = 'https://' + host;
+
 const bucket = 'example-bucket';
-const objectKey = 'example-object'
-const expiration = 86400  // time in seconds
+const objectKey = 'example-object';
+
+const expiration = 86400;  // time in seconds
 
 // hashing and signing methods
 function hash(key, msg) {
-    var hmac = crypto.createHmac('sha256', key);
-    hmac.update(msg, 'utf8');
-    return hmac.digest();
+	let hmac = crypto.createHmac('sha256', key);
+	hmac.update(msg, 'utf8');
+	return hmac.digest();
 }
 
 function hmacHex(key, msg) {
-    var hmac = crypto.createHmac('sha256', key);
-    hmac.update(msg, 'utf8');
-    return hmac.digest('hex');
+	let hmac = crypto.createHmac('sha256', key);
+	hmac.update(msg, 'utf8');
+	return hmac.digest('hex');
 }
 
 function hashHex(msg) {
-    var hash = crypto.createHash('sha256');
-    hash.update(msg);
-    return hash.digest('hex');
+	let hash = crypto.createHash('sha256');
+	hash.update(msg);
+	return hash.digest('hex');
 }
 
 // region is a wildcard value that takes the place of the AWS region value
 // as COS doesn't use the same conventions for regions, this parameter can accept any string
 function createSignatureKey(key, datestamp, region, service) {
-    keyDate = hash(('AWS4' + key), datestamp);
-    keyString = hash(keyDate, region);
-    keyService = hash(keyString, service);
-    keySigning = hash(keyService, 'aws4_request');
-    return keySigning;
+	keyDate = hash(('AWS4' + key), datestamp);
+	keyString = hash(keyDate, region);
+	keyService = hash(keyString, service);
+	keySigning = hash(keyService, 'aws4_request');
+	return keySigning;
 }
 
 function createHexSignatureKey(key, datestamp, region, service) {
-    keyDate = hashHex(('AWS4' + key), datestamp);
-    keyString = hashHex(keyDate, region);
-    keyService = hashHex(keyString, service);
-    keySigning = hashHex(keyService, 'aws4_request');
-    return keySigning;
+	keyDate = hashHex(('AWS4' + key), datestamp);
+	keyString = hashHex(keyDate, region);
+	keyService = hashHex(keyString, service);
+	keySigning = hashHex(keyService, 'aws4_request');
+	return keySigning;
 }
 
 function printDebug() {
@@ -152,13 +157,13 @@ console.log(`\nSending ${httpMethod} request to IBM COS -----------------------`
 console.log('Request URL = ' + requestUrl);
 
 var request = https.get(requestUrl, function (response) {
-    console.log('\nResponse from IBM COS ----------------------------------');
-    console.log(`Response code: ${response.statusCode}\n`);
+	console.log('\nResponse from IBM COS ----------------------------------');
+	console.log(`Response code: ${response.statusCode}\n`);
 
-    response.on('data', function (chunk) {
-        console.log('Response: ' + chunk);
-        printDebug();
-    });
+	response.on('data', function (chunk) {
+		console.log('Response: ' + chunk);
+		printDebug();
+	});
 });
 
 request.end();
diff --git a/src/gcp/README.MD b/src/gcp/README.MD
@@ -1,76 +1,51 @@
-# ImageAPI
-This project is experimenting with different providers of cloud based web hooks and storage.  
-[Note: This is in the very early stages of the experimentation and is not currently functional. The specs and approach may change without any notice.]  
+# ImageAPI-GCP
+This sub-project implements a basic Image management API in GCP. (index.js)   
 
-The base use case is the ability to upload remote images into a cloud storage bucket as well as to retrieve and resize them.
-(Note: some providers have resizing capabilities inbuilt [eg. GCP], this project will examine other methods to implement the same)
+Please note: `index.js` contains the latest functioning code. `experiments/` is from my initial experimentation and may not be functional.
 
-## API Requirements
-- Upload Images via URL
-- Download Images / provide Signed URL
-- Resize Images
-- 80 day expiry
-
-### Additional Considerations
-- Real time updates for large uploads
-- Performance [Optimization for write heavy load]
-- Caching
-- Security [Who can access which images]
-- Resilience []
+## Current State
+The first phase on this module has been completed.
+This encompasses the basic functionality of uploading a remote image (via stream), and providing a signed URL by which to access that image. It consists of one REST end point.
 
 ## Setup
 1. Create Project and add Project_ID to config.json   
 2. Create a service account with access to upload Google Functions and save the key in keys/gcp_functions.json   
 3. Create Bucket in Google Cloud Storage and update bucket_name in config.json  
 4. Create a service account with write permissions to your google storage bucket and save the key in keys/gcp_storage.json  
-TODO: add script to deploy.sh to add Project_ID to bucket name
-
-
-### Service Accounts
-
-## Leads
-Consider spinning up app in Firebase
-Resumable file uploads: https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload
-https://github.com/mkahn5/gcloud-resumable-uploads/blob/master/views/index.ejs
-Google Websocket demo: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/tree/master/appengine/websockets
-Firebase approach to uploads: https://firebase.google.com/docs/storage/web/upload-files#monitor_upload_progress
-Pub/Sub: https://github.com/googleapis/nodejs-pubsub/
-
 
 ## TODO
 - Filetype verification
 - Partitioned file uploads
 - Caching
-	- Match Catch expiry to Bucket item expiration
-
-## Architecture
-The API endpoint footprint is small and doesn't need to maintain any state. This makes it a prime candidate for Lamba-style webhooks. (Benefits: Auto-scaling, minimal upkeep, smaller code footprint etc.)  
-User will interact with a single REST endpoint (/images) through GET and PUT requests.  
-
-![Image API GET sequence](/doc_assets/GET.png)  
+	- Match Cache expiry to Bucket item expiration
+- Add script to deploy.sh to add Project_ID to bucket name
+- Security and signing
 
-![Image API PUT sequence](/doc_assets/PUT.png)
-
-
-### Provider
-Note: GCP's Node 8 driver is still Beta.
-
-### Implementation
+## Implementation Notes
 #### Item Expiry
 The item expiry can be handled directly in the data bucket policy.
 `gsutil mb --retention 80d gs://ImageAPI`
 (For a more nuanced/extensible approach we could specify a more detailed lifecycle policy or have a lambda function launched via scheduled CRON job - eg. via cloud scheduler)
 
+### GCP's Node 8 driver is still Beta.  
 
-## Implementation Notes
-### Code/Folder Structure - gcloud deploy limitations
+### Code/Folder Structure - gcloud deploy limitations  
 The `gcloud deploy` command, unfortunately, appears to only work on a singular local index.js file (or remote repositories).
 
-### (Prepackaged solutions)
+### (Prepackaged solutions)  
 Google's AppEngine provides ready made image servicing, including resizing and cropping.
 https://cloud.google.com/appengine/docs/standard/python/refdocs/google.appengine.api.images#google.appengine.api.images.get_serving_url
 https://medium.com/google-cloud/uploading-resizing-and-serving-images-with-google-cloud-platform-ca9631a2c556
 
 ### Performance considerations
 #### HTTP Requests to Google Cloud
 We're currently utilizing Google's prebuilt node module for access. For a smaller footprint, to aid with speed of load for the cloud functions, we could use a streamlined request library to interact directly with their REST API.  
+
+
+## Leads for future research
+Consider spinning up app in Firebase
+Resumable file uploads: https://cloud.google.com/storage/docs/json_api/v1/how-tos/resumable-upload
+https://github.com/mkahn5/gcloud-resumable-uploads/blob/master/views/index.ejs
+Google Websocket demo: https://github.com/GoogleCloudPlatform/nodejs-docs-samples/tree/master/appengine/websockets
+Firebase approach to uploads: https://firebase.google.com/docs/storage/web/upload-files#monitor_upload_progress
+Pub/Sub: https://github.com/googleapis/nodejs-pubsub/
