Added kafka-consumer support.

Greg Palmer · Greg Palmer · commit 901636c9fd63 · 2017-12-05T09:38:24.000-05:00
diff --git a/resources/Mesosphere-SMACK-Stack-Tutorial.pdf b/resources/Mesosphere-SMACK-Stack-Tutorial.pdf
diff --git a/scripts/run-sample-spark-kafka-job.sh b/scripts/run-sample-spark-kafka-job.sh
@@ -3,6 +3,6 @@
 # SCRIPT: run-sample-spark-kafka-job.sh
 #
 
-dcos spark run --name 'spark' --submit-args='--conf spark.eventLog.enabled=true --conf spark.eventLog.dir=hdfs://hdfs/history --conf Dspark.mesos.coarse=true --conf spark.cores.max=4 --conf spark.executor.memory=1g --driver-cores 1 --driver-memory 1g --class HDFSWordCount http://infinity-artifacts.s3.amazonaws.com/spark/sparkjob-assembly-1.0.jar hdfs:///test-data/test-data.txt'
+dcos spark run --submit-args='--conf spark.eventLog.enabled=true --conf spark.eventLog.dir=hdfs://hdfs/history --conf Dspark.mesos.coarse=true --conf spark.cores.max=4 --conf spark.executor.memory=1g --driver-cores 1 --driver-memory 1g --class org.apache.spark.examples.streaming.KafkaWordCount  https://downloads.mesosphere.com/spark/assets/spark-examples_2.10-1.4.0-SNAPSHOT.jar mesos://leader.mesos:5050 zk-1.zk,zk-2.zk,zk-3.zk my-consumer-group my-topic 1'
 
 # End of script
diff --git a/scripts/start-smackstack.sh b/scripts/start-smackstack.sh
@@ -200,15 +200,43 @@ do
     sleep 10
 done
 
+echo
+echo " #####################################"
+echo " ### Starting Kafka Consumer       ###"
+echo " #####################################"
+echo
+echo
+echo " Waiting for Kafka service to start. "
+while true 
+do
+    task_status=$(dcos task |grep 'kafka-0-broker ' | awk '{print $4}')
+
+    if [ "$task_status" != "R" ]
+    then
+        printf "."
+    else
+        echo " Kafka service is running."
+        break
+    fi
+    sleep 10
+done
+
+dcos marathon app add config/kafka-consumer.json
+
+sleep 5
+
 echo
 echo " #############################################################"
-echo " ### Smack Stack start up complete.                        ###"
+echo " ### SMACK Stack start up complete.                        ###"
 echo " ### If you would like to run a Spark job that reads       ###"
-echo " ### from the HDFS file system, run the following command: ###"
+echo " ### from the HDFS file system or from a Kafka queue,      ###"
+echo " ### run the following commands:                           ###"
 echo " #############################################################"
 echo
 
 echo "     $ scripts/run-sample-spark-hdfs-job.sh "
+echo
+echo "     $ scripts/run-sample-spark-kafka-job.sh "
 
 echo
 
diff --git a/scripts/stop-smackstack.sh b/scripts/stop-smackstack.sh
@@ -36,6 +36,11 @@ echo " ### Stopping HDFS "
 sleep 2
 dcos package uninstall hdfs --app-id='hdfs' --yes
 
+echo
+echo " ### Stopping Kafka Consumer "
+
+dcos marathon app remove kafka-consumer
+
 echo
 echo " ### Stopping Kafka "
 
@@ -50,7 +55,6 @@ echo " ### Stopping Cassandra "
 sleep 2
 dcos package uninstall cassandra --app-id='cassandra' --yes
 
-
 # Zookeeper cleanup
 # First, make sure all services are removed
 

Original file line number	Diff line number	Diff line change
`@@ -3,6 +3,6 @@`
`3`	`3`	`# SCRIPT: run-sample-spark-kafka-job.sh`
`4`	`4`	`#`
`5`	`5`
`6`		`-dcos spark run --name 'spark' --submit-args='--conf spark.eventLog.enabled=true --conf spark.eventLog.dir=hdfs://hdfs/history --conf Dspark.mesos.coarse=true --conf spark.cores.max=4 --conf spark.executor.memory=1g --driver-cores 1 --driver-memory 1g --class HDFSWordCount http://infinity-artifacts.s3.amazonaws.com/spark/sparkjob-assembly-1.0.jar hdfs:///test-data/test-data.txt'`
	`6`	`+dcos spark run --submit-args='--conf spark.eventLog.enabled=true --conf spark.eventLog.dir=hdfs://hdfs/history --conf Dspark.mesos.coarse=true --conf spark.cores.max=4 --conf spark.executor.memory=1g --driver-cores 1 --driver-memory 1g --class org.apache.spark.examples.streaming.KafkaWordCount https://downloads.mesosphere.com/spark/assets/spark-examples_2.10-1.4.0-SNAPSHOT.jar mesos://leader.mesos:5050 zk-1.zk,zk-2.zk,zk-3.zk my-consumer-group my-topic 1'`
`7`	`7`
`8`	`8`	`# End of script`