Skip to content

Commit

Permalink
upgrade spark 3.5.3 & jdbc postgres driver
Browse files Browse the repository at this point in the history
  • Loading branch information
ihiverlet committed Oct 18, 2024
1 parent cab4853 commit 87a3fe6
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 9 deletions.
10 changes: 5 additions & 5 deletions .github/workflows/main-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ jobs:
base_image: python-minimal
python_version_1: 3.12.6
python_version_2: 3.11.10
spark_version: 3.5.2
spark_version: 3.5.3
build_gpu: false
secrets: inherit
jupyter-pyspark:
Expand All @@ -115,7 +115,7 @@ jobs:
base_image: pyspark
python_version_1: 3.12.6
python_version_2: 3.11.10
spark_version: 3.5.2
spark_version: 3.5.3
build_gpu: false
secrets: inherit
vscode-pyspark:
Expand All @@ -127,7 +127,7 @@ jobs:
base_image: pyspark
python_version_1: 3.12.6
python_version_2: 3.11.10
spark_version: 3.5.2
spark_version: 3.5.3
build_gpu: false
secrets: inherit
jupyter-pytorch:
Expand Down Expand Up @@ -189,7 +189,7 @@ jobs:
base_image: r-minimal
r_version_1: 4.4.1
r_version_2: 4.3.3
spark_version: 3.5.2
spark_version: 3.5.3
build_gpu: false
secrets: inherit
rstudio-sparkr:
Expand All @@ -201,7 +201,7 @@ jobs:
base_image: sparkr
r_version_1: 4.4.1
r_version_2: 4.3.3
spark_version: 3.5.2
spark_version: 3.5.3
build_gpu: false
secrets: inherit
jupyter-r:
Expand Down
9 changes: 6 additions & 3 deletions scripts/install-spark-hadoop-hive.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash
set -e

HADOOP_VERSION="3.3.6"
HADOOP_VERSION="3.4.0"
HIVE_VERSION="2.3.9"
HIVE_LISTENER_VERSION="0.0.3"

Expand Down Expand Up @@ -37,12 +37,15 @@ wget -q ${SPARK_BUILD_S3_BUCKET}/hive-listener/${HIVE_LISTENER_JAR}
mv ${HIVE_LISTENER_JAR} ${HIVE_HOME}/lib/hive-listener.jar

# Add postgreSQL support to Hive
wget -q https://jdbc.postgresql.org/download/postgresql-42.2.18.jar
mv postgresql-42.2.18.jar ${HIVE_HOME}/lib/postgresql-jdbc.jar
wget -q https://jdbc.postgresql.org/download/postgresql-42.7.3.jar
mv postgresql-42.7.3.jar ${HIVE_HOME}/lib/postgresql-jdbc.jar

# Fix versions inconsistencies of some binaries between Hadoop & Hive distributions
rm ${HIVE_HOME}/lib/guava-14.0.1.jar
cp ${HADOOP_HOME}/share/hadoop/common/lib/guava-27.0-jre.jar ${HIVE_HOME}/lib/
wget -q https://repo1.maven.org/maven2/jline/jline/2.14.6/jline-2.14.6.jar
mv jline-2.14.6.jar ${HIVE_HOME}/lib/
rm ${HIVE_HOME}/lib/jline-2.12.jar

# Fix multiple bindings
rm ${HADOOP_HOME}/share/hadoop/tools/lib/bundle-2.23.19.jar
2 changes: 1 addition & 1 deletion spark/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ LABEL maintainer="InseeFrLab <innovation@insee.fr>"

ARG BASE_IMAGE

ARG SPARK_VERSION="3.5.1"
ARG SPARK_VERSION="3.5.3"
ENV SPARK_VERSION=${SPARK_VERSION}
ENV JAVA_VERSION="17"

Expand Down

0 comments on commit 87a3fe6

Please sign in to comment.