Refactoring to the right multi-project sbt structure

SemyonSinchenko · Jul 17, 2025 · Jul 17, 2025 · Jul 17, 2025 · Jul 17, 2025 · Jul 17, 2025
commit c85024bdaa7e873f7c0dca8d830e2886ddfbea8a
diff --git a/build.sbt b/build.sbt
@@ -100,7 +100,7 @@ lazy val commonSetting = Seq(
     else
       "-Wunused:imports"))
 
-lazy val root = (project in file("."))
+lazy val core = (project in file("graphframes-core"))
   .settings(
     commonSetting,
     name := "graphframes",
@@ -123,7 +123,7 @@ lazy val root = (project in file("."))
     Compile / packageSrc / publishArtifact := true)
 
 lazy val connect = (project in file("graphframes-connect"))
-  .dependsOn(root)
+  .dependsOn(core)
   .settings(
     name := s"graphframes-connect",
     moduleName := s"${name.value}-spark${sparkMajorVer}",

diff --git a/...he/spark/sql/graphframes/SparkShims.scala → ...he/spark/sql/graphframes/SparkShims.scala b/...he/spark/sql/graphframes/SparkShims.scala → ...he/spark/sql/graphframes/SparkShims.scala
diff --git a/...he/spark/sql/graphframes/SparkShims.scala → ...he/spark/sql/graphframes/SparkShims.scala b/...he/spark/sql/graphframes/SparkShims.scala → ...he/spark/sql/graphframes/SparkShims.scala
diff --git a/...ark/sql/graphframes/GraphFramesConf.scala → ...ark/sql/graphframes/GraphFramesConf.scala b/...ark/sql/graphframes/GraphFramesConf.scala → ...ark/sql/graphframes/GraphFramesConf.scala
diff --git a/...in/scala/org/graphframes/GraphFrame.scala → ...in/scala/org/graphframes/GraphFrame.scala b/...in/scala/org/graphframes/GraphFrame.scala → ...in/scala/org/graphframes/GraphFrame.scala
diff --git a/...org/graphframes/GraphFramePythonAPI.scala → ...org/graphframes/GraphFramePythonAPI.scala b/...org/graphframes/GraphFramePythonAPI.scala → ...org/graphframes/GraphFramePythonAPI.scala
diff --git a/src/main/scala/org/graphframes/Logging.scala → .../main/scala/org/graphframes/Logging.scala b/src/main/scala/org/graphframes/Logging.scala → .../main/scala/org/graphframes/Logging.scala
diff --git a/...phframes/examples/BeliefPropagation.scala → ...phframes/examples/BeliefPropagation.scala b/...phframes/examples/BeliefPropagation.scala → ...phframes/examples/BeliefPropagation.scala
diff --git a/...ala/org/graphframes/examples/Graphs.scala → ...ala/org/graphframes/examples/Graphs.scala b/...ala/org/graphframes/examples/Graphs.scala → ...ala/org/graphframes/examples/Graphs.scala
diff --git a/.../org/graphframes/examples/LDBCUtils.scala → .../org/graphframes/examples/LDBCUtils.scala b/.../org/graphframes/examples/LDBCUtils.scala → .../org/graphframes/examples/LDBCUtils.scala
diff --git a/...in/scala/org/graphframes/exceptions.scala → ...in/scala/org/graphframes/exceptions.scala b/...in/scala/org/graphframes/exceptions.scala → ...in/scala/org/graphframes/exceptions.scala
diff --git a/...g/graphframes/lib/AggregateMessages.scala → ...g/graphframes/lib/AggregateMessages.scala b/...g/graphframes/lib/AggregateMessages.scala → ...g/graphframes/lib/AggregateMessages.scala
diff --git a/src/main/scala/org/graphframes/lib/BFS.scala → .../main/scala/org/graphframes/lib/BFS.scala b/src/main/scala/org/graphframes/lib/BFS.scala → .../main/scala/org/graphframes/lib/BFS.scala
diff --git a/...graphframes/lib/ConnectedComponents.scala → ...graphframes/lib/ConnectedComponents.scala b/...graphframes/lib/ConnectedComponents.scala → ...graphframes/lib/ConnectedComponents.scala
diff --git a/...g/graphframes/lib/GraphXConversions.scala → ...g/graphframes/lib/GraphXConversions.scala b/...g/graphframes/lib/GraphXConversions.scala → ...g/graphframes/lib/GraphXConversions.scala
diff --git a/...rg/graphframes/lib/LabelPropagation.scala → ...rg/graphframes/lib/LabelPropagation.scala b/...rg/graphframes/lib/LabelPropagation.scala → ...rg/graphframes/lib/LabelPropagation.scala
diff --git a/.../scala/org/graphframes/lib/PageRank.scala → .../scala/org/graphframes/lib/PageRank.scala b/.../scala/org/graphframes/lib/PageRank.scala → .../scala/org/graphframes/lib/PageRank.scala
diff --git a/...es/lib/ParallelPersonalizedPageRank.scala → ...es/lib/ParallelPersonalizedPageRank.scala b/...es/lib/ParallelPersonalizedPageRank.scala → ...es/lib/ParallelPersonalizedPageRank.scala
diff --git a/...in/scala/org/graphframes/lib/Pregel.scala → ...in/scala/org/graphframes/lib/Pregel.scala b/...in/scala/org/graphframes/lib/Pregel.scala → ...in/scala/org/graphframes/lib/Pregel.scala
diff --git a/...ala/org/graphframes/lib/SVDPlusPlus.scala → ...ala/org/graphframes/lib/SVDPlusPlus.scala b/...ala/org/graphframes/lib/SVDPlusPlus.scala → ...ala/org/graphframes/lib/SVDPlusPlus.scala
diff --git a/...a/org/graphframes/lib/ShortestPaths.scala → ...a/org/graphframes/lib/ShortestPaths.scala b/...a/org/graphframes/lib/ShortestPaths.scala → ...a/org/graphframes/lib/ShortestPaths.scala
diff --git a/...mes/lib/StronglyConnectedComponents.scala → ...mes/lib/StronglyConnectedComponents.scala b/...mes/lib/StronglyConnectedComponents.scala → ...mes/lib/StronglyConnectedComponents.scala
diff --git a/...a/org/graphframes/lib/TriangleCount.scala → ...a/org/graphframes/lib/TriangleCount.scala b/...a/org/graphframes/lib/TriangleCount.scala → ...a/org/graphframes/lib/TriangleCount.scala
diff --git a/src/main/scala/org/graphframes/mixins.scala → ...c/main/scala/org/graphframes/mixins.scala b/src/main/scala/org/graphframes/mixins.scala → ...c/main/scala/org/graphframes/mixins.scala
diff --git a/...la/org/graphframes/pattern/patterns.scala → ...la/org/graphframes/pattern/patterns.scala b/...la/org/graphframes/pattern/patterns.scala → ...la/org/graphframes/pattern/patterns.scala
diff --git a/src/test/resources/als-test.data → ...mes-core/src/test/resources/als-test.data b/src/test/resources/als-test.data → ...mes-core/src/test/resources/als-test.data
diff --git a/src/test/resources/log4j.properties → ...-core/src/test/resources/log4j.properties b/src/test/resources/log4j.properties → ...-core/src/test/resources/log4j.properties
diff --git a/src/test/resources/log4j2.properties → ...core/src/test/resources/log4j2.properties b/src/test/resources/log4j2.properties → ...core/src/test/resources/log4j2.properties
diff --git a/...ala/org/graphframes/GraphFrameSuite.scala → ...ala/org/graphframes/GraphFrameSuite.scala b/...ala/org/graphframes/GraphFrameSuite.scala → ...ala/org/graphframes/GraphFrameSuite.scala
diff --git a/...phframes/GraphFrameTestSparkContext.scala → ...phframes/GraphFrameTestSparkContext.scala b/...phframes/GraphFrameTestSparkContext.scala → ...phframes/GraphFrameTestSparkContext.scala
diff --git a/...a/org/graphframes/PatternMatchSuite.scala → ...a/org/graphframes/PatternMatchSuite.scala b/...a/org/graphframes/PatternMatchSuite.scala → ...a/org/graphframes/PatternMatchSuite.scala
diff --git a/...scala/org/graphframes/SparkFunSuite.scala → ...scala/org/graphframes/SparkFunSuite.scala b/...scala/org/graphframes/SparkFunSuite.scala → ...scala/org/graphframes/SparkFunSuite.scala
diff --git a/...est/scala/org/graphframes/TestUtils.scala → ...est/scala/org/graphframes/TestUtils.scala b/...est/scala/org/graphframes/TestUtils.scala → ...est/scala/org/graphframes/TestUtils.scala
diff --git a/...mes/examples/BeliefPropagationSuite.scala → ...mes/examples/BeliefPropagationSuite.scala b/...mes/examples/BeliefPropagationSuite.scala → ...mes/examples/BeliefPropagationSuite.scala
diff --git a/...rg/graphframes/examples/GraphsSuite.scala → ...rg/graphframes/examples/GraphsSuite.scala b/...rg/graphframes/examples/GraphsSuite.scala → ...rg/graphframes/examples/GraphsSuite.scala
diff --git a/.../org/graphframes/ldbc/TestLDBCCases.scala → .../org/graphframes/ldbc/TestLDBCCases.scala b/.../org/graphframes/ldbc/TestLDBCCases.scala → .../org/graphframes/ldbc/TestLDBCCases.scala
diff --git a/...phframes/lib/AggregateMessagesSuite.scala → ...phframes/lib/AggregateMessagesSuite.scala b/...phframes/lib/AggregateMessagesSuite.scala → ...phframes/lib/AggregateMessagesSuite.scala
diff --git a/.../scala/org/graphframes/lib/BFSSuite.scala → .../scala/org/graphframes/lib/BFSSuite.scala b/.../scala/org/graphframes/lib/BFSSuite.scala → .../scala/org/graphframes/lib/BFSSuite.scala
diff --git a/...frames/lib/ConnectedComponentsSuite.scala → ...frames/lib/ConnectedComponentsSuite.scala b/...frames/lib/ConnectedComponentsSuite.scala → ...frames/lib/ConnectedComponentsSuite.scala
diff --git a/...aphframes/lib/LabelPropagationSuite.scala → ...aphframes/lib/LabelPropagationSuite.scala b/...aphframes/lib/LabelPropagationSuite.scala → ...aphframes/lib/LabelPropagationSuite.scala
diff --git a/...a/org/graphframes/lib/PageRankSuite.scala → ...a/org/graphframes/lib/PageRankSuite.scala b/...a/org/graphframes/lib/PageRankSuite.scala → ...a/org/graphframes/lib/PageRankSuite.scala
diff --git a/...b/ParallelPersonalizedPageRankSuite.scala → ...b/ParallelPersonalizedPageRankSuite.scala b/...b/ParallelPersonalizedPageRankSuite.scala → ...b/ParallelPersonalizedPageRankSuite.scala
diff --git a/...ala/org/graphframes/lib/PregelSuite.scala → ...ala/org/graphframes/lib/PregelSuite.scala b/...ala/org/graphframes/lib/PregelSuite.scala → ...ala/org/graphframes/lib/PregelSuite.scala
diff --git a/...rg/graphframes/lib/SVDPlusPlusSuite.scala → ...rg/graphframes/lib/SVDPlusPlusSuite.scala b/...rg/graphframes/lib/SVDPlusPlusSuite.scala → ...rg/graphframes/lib/SVDPlusPlusSuite.scala
diff --git a/.../graphframes/lib/ShortestPathsSuite.scala → .../graphframes/lib/ShortestPathsSuite.scala b/.../graphframes/lib/ShortestPathsSuite.scala → .../graphframes/lib/ShortestPathsSuite.scala
diff --git a/...ib/StronglyConnectedComponentsSuite.scala → ...ib/StronglyConnectedComponentsSuite.scala b/...ib/StronglyConnectedComponentsSuite.scala → ...ib/StronglyConnectedComponentsSuite.scala
diff --git a/.../graphframes/lib/TriangleCountSuite.scala → .../graphframes/lib/TriangleCountSuite.scala b/.../graphframes/lib/TriangleCountSuite.scala → .../graphframes/lib/TriangleCountSuite.scala
diff --git a/...rg/graphframes/pattern/PatternSuite.scala → ...rg/graphframes/pattern/PatternSuite.scala b/...rg/graphframes/pattern/PatternSuite.scala → ...rg/graphframes/pattern/PatternSuite.scala
diff --git a/python/dev/build_jar.py b/python/dev/build_jar.py
@@ -15,8 +15,8 @@ def build(spark_versions: Sequence[str] = ["3.5.5"]):
         sbt_build_command = [
             sbt_executable,
             f"-Dspark.version={spark_version}",
-            "clean",
-            "package",
+            "core/clean",
+            "core/package",
             "connect/clean",
             "connect/package"
         ]

diff --git a/python/tests/conftest.py b/python/tests/conftest.py
@@ -30,7 +30,7 @@ def get_gf_jar_locations() -> Tuple[str, str]:
     this function will raise an exception!
     """
     project_root = pathlib.Path(__file__).parent.parent.parent
-    core_dir = project_root / "target" / f"scala-{scala_version}"
+    core_dir = project_root / "graphframes-core" / "target" / f"scala-{scala_version}"
     connect_dir = project_root / "graphframes-connect" / "target" / f"scala-{scala_version}"
 
     core_jar: Optional[str] = None