graphframes · rjurney · Feb 16, 2025 · Feb 16, 2025 · Feb 16, 2025 · Feb 16, 2025
diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
@@ -7,8 +7,8 @@ jobs:
       matrix:
         include:
           - spark-version: 3.5.4
-            scala-version: 2.12.18
-            python-version: 3.9.19
+            scala-version: 2.12.20
+            python-version: 3.11.11
     runs-on: ubuntu-22.04
     env:
       # define Java options for both official sbt and sbt-extras
@@ -35,8 +35,11 @@ jobs:
       run: |
         python -m pip install --upgrade pip wheel
         pip install -r ./python/requirements.txt
+        pip install -r ./python/requirements-dev.txt
         pip install pyspark==${{ matrix.spark-version }}
     - name: Test
       run: |
+        python python/setup.py install
+        python python/setup.py bdist_wheel
         export SPARK_HOME=$(python -c "import os; from importlib.util import find_spec; print(os.path.join(os.path.dirname(find_spec('pyspark').origin)))")
         ./python/run-tests.sh
diff --git a/.gitignore b/.gitignore
@@ -26,3 +26,12 @@ project/plugins/project/
 
 # Mac
 *.DS_Store
+
+# Python specific
+python/build
+python/dist
+build/lib
+python/graphframes.egg-info
+python/graphframes/tutorials/data
+python/docs/_build
+python/docs/_site
diff --git a/Dockerfile b/Dockerfile
@@ -1,16 +1,16 @@
 FROM ubuntu:22.04
 
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.9
 ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && \
-    apt-get install -y wget bzip2 build-essential openjdk-8-jdk ssh sudo && \
+    apt-get install -y wget bzip2 build-essential openjdk-11-jdk ssh sudo && \
     apt-get clean
 
 # Install Spark and update env variables.
-ENV SCALA_VERSION 2.12.17
-ENV SPARK_VERSION "3.4.1"
-ENV SPARK_BUILD "spark-${SPARK_VERSION}-bin-hadoop3.2"
+ENV SCALA_VERSION 2.12.20
+ENV SPARK_VERSION "3.5.4"
+ENV SPARK_BUILD "spark-${SPARK_VERSION}-bin-hadoop3"
 ENV SPARK_BUILD_URL "https://dist.apache.org/repos/dist/release/spark/spark-${SPARK_VERSION}/${SPARK_BUILD}.tgz"
 RUN wget --quiet "$SPARK_BUILD_URL" -O /tmp/spark.tgz && \
     tar -C /opt -xf /tmp/spark.tgz && \

diff --git a/build.sbt b/build.sbt
@@ -3,7 +3,7 @@ import ReleaseTransformations._
 lazy val sparkVer = sys.props.getOrElse("spark.version", "3.5.4")
 lazy val sparkBranch = sparkVer.substring(0, 3)
 lazy val defaultScalaVer = sparkBranch match {
-  case "3.5" => "2.12.18"
+  case "3.5" => "2.12.20"
   case _ => throw new IllegalArgumentException(s"Unsupported Spark version: $sparkVer.")
 }
 lazy val scalaVer = sys.props.getOrElse("scala.version", defaultScalaVer)

diff --git a/docs/_config.yml b/docs/_config.yml
@@ -13,7 +13,7 @@ include:
 
 # These allow the documentation to be updated with newer releases
 # of Spark, Scala, and Mesos.
-GRAPHFRAMES_VERSION: 0.8.4
+GRAPHFRAMES_VERSION: 0.8.5
 #SCALA_BINARY_VERSION: "2.10"
 #SCALA_VERSION: "2.10.4"
 #MESOS_VERSION: 0.21.0

diff --git a/python/MANIFEST.in b/python/MANIFEST.in
@@ -2,3 +2,7 @@
 # https://github.com/pypa/sampleproject/blob/master/MANIFEST.in
 # For more details about the MANIFEST file, you may read the docs at
 # https://docs.python.org/2/distutils/sourcedist.html#the-manifest-in-template
+recursive-include python/graphframes *.py
+recursive-exclude * __pycache__
+recursive-exclude * *.pyc
+include graphframes/tutorials/data/.exists