diff --git a/Makefile b/Makefile index 36f762c..dc44979 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ up: docker network create spark-net docker-compose build - docker-compose up + docker-compose up -d down: docker-compose down @@ -12,8 +12,8 @@ bash: run: docker build -t zeppelin ./zeppelin/. - docker run -it --rm --net spark-net -p 80:8080 -v $(shell pwd)/notebook:/opt/zeppelin/notebook -v $(shell pwd)/zeppelin-0.7.2-bin-all:/opt/zeppelin zeppelin /bin/bash + docker run -it --rm --net spark-net -p 80:8080 -v $(shell pwd)/notebook:/opt/zeppelin/notebook -v $(shell pwd)/zeppelin-0.8.0-bin-all:/opt/zeppelin zeppelin /bin/bash #docker run -it --rm --net spark-net -p 80:8080 -v $(shell pwd)/notebook:/opt/zeppelin/notebook zeppelin /opt/zeppelin/bin/zeppelin.sh build: - docker build -t earthquakesan/zeppelin:0.7.2 ./zeppelin/. + docker build -t bde2020/zeppelin:0.8.0 ./zeppelin/. diff --git a/docker-compose.yml b/docker-compose.yml index 447d604..51543ca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - CORE_CONF_fs_defaultFS=hdfs://namenode:8020 healthcheck: interval: 5s - retries: 100 + retries: 100 networks: - spark-net datanode: @@ -22,15 +22,11 @@ services: environment: - CORE_CONF_fs_defaultFS=hdfs://namenode:8020 depends_on: - namenode: - condition: service_healthy - healthcheck: - interval: 5s - retries: 100 + - namenode networks: - spark-net spark-master: - image: bde2020/spark-master:2.1.0-hadoop2.8-hive-java8 + image: bde2020/spark-master:2.3.1-hadoop2.8 container_name: spark-master ports: - "8080:8080" @@ -38,27 +34,18 @@ services: environment: - CORE_CONF_fs_defaultFS=hdfs://namenode:8020 depends_on: - namenode: - condition: service_healthy - datanode: - condition: service_healthy - healthcheck: - interval: 5s - retries: 100 + - namenode + - datanode networks: - spark-net spark-worker: - image: bde2020/spark-worker:2.1.0-hadoop2.8-hive-java8 + image: bde2020/spark-worker:2.3.1-hadoop2.8 environment: - "SPARK_MASTER=spark://spark-master:7077" environment: - CORE_CONF_fs_defaultFS=hdfs://namenode:8020 depends_on: - spark-master: - condition: service_healthy - healthcheck: - interval: 5s - retries: 100 + - spark-master networks: - spark-net zeppelin: @@ -73,10 +60,8 @@ services: MASTER: "spark://spark-master:7077" #SPARK_SUBMIT_OPTIONS: "--jars /opt/sansa-examples/jars/sansa-examples-spark-2016-12.jar" depends_on: - spark-master: - condition: service_healthy - namenode: - condition: service_healthy + - spark-master + - namenode networks: - spark-net diff --git a/zeppelin/Dockerfile b/zeppelin/Dockerfile index 8d61b58..978a995 100644 --- a/zeppelin/Dockerfile +++ b/zeppelin/Dockerfile @@ -1,18 +1,19 @@ -FROM bde2020/spark-base:2.1.0-hadoop2.8-hive-java8 +FROM bde2020/spark-base:2.3.1-hadoop2.8 MAINTAINER Ivan Ermilov +MAINTAINER Gezim Sejdiu -ENV APACHE_SPARK_VERSION 2.1.0 +ENV APACHE_SPARK_VERSION 2.3.1 ENV APACHE_HADOOP_VERSION 2.8.0 -ENV ZEPPELIN_VERSION 0.7.2 +ENV ZEPPELIN_VERSION 0.8.0 RUN apt-get update && apt-get install wget RUN set -x \ - && curl -fSL "http://www-eu.apache.org/dist/zeppelin/zeppelin-0.7.2/zeppelin-0.7.2-bin-all.tgz" -o /tmp/zeppelin.tgz \ + && curl -fSL "http://www-eu.apache.org/dist/zeppelin/zeppelin-${ZEPPELIN_VERSION}/zeppelin-${ZEPPELIN_VERSION}-bin-all.tgz" -o /tmp/zeppelin.tgz \ && tar -xzvf /tmp/zeppelin.tgz -C /opt/ \ && mv /opt/zeppelin-* /opt/zeppelin \ && rm /tmp/zeppelin.tgz -ENV SPARK_SUBMIT_OPTIONS "--jars /opt/zeppelin/sansa-examples-spark-2016-12.jar" +ENV SPARK_SUBMIT_OPTIONS "--jars /opt/zeppelin/sansa-examples-spark-2018-06.jar" WORKDIR /opt/zeppelin