Skip to content

Commit c091c70

Browse files
mshang816mbasmanova
mshang816
authored and
mbasmanova
committed
Fix the out of disc space issue
Fix the out of disc space issue by removing caching for maven. First we build all dependents of presto-native-execution every time which only take around 2+ minutes with "-am" instead of building many component not used. Also remove caching for maven as maven build only takes 2 mins now. This will save us some space to avoid the disc full issue. For some reason, tests would fail if they are run together. So as a work-around we run each Test class in its separate commands. Temporarily disable flaky TestPrestoSparkNativeGeneralQueries.testUnionAll() Test plan - Make all CI runs are green
1 parent 4145bd2 commit c091c70

2 files changed

Lines changed: 23 additions & 33 deletions

File tree

‎.github/workflows/test-native.yml‎

Lines changed: 18 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ on:
1717
env:
1818
MAVEN_OPTS: "-Xmx1024M -XX:+ExitOnOutOfMemoryError"
1919
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
20-
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
20+
MAVEN_FAST_INSTALL: "-B -V -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
2121
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
2222

2323
jobs:
@@ -35,36 +35,30 @@ jobs:
3535

3636
steps:
3737
- uses: actions/checkout@v3
38-
- uses: actions/setup-java@v1
39-
with:
40-
java-version: 8
4138

42-
- name: Get date for ccache
43-
id: get-date
44-
run: |
45-
echo "::set-output name=date::$(/bin/date -u "+%Y%m%d")"
46-
shell: bash
39+
- name: Get key for ccache
40+
id: get-ccache-key
41+
run: echo "CCACHE_KEY=$(/bin/date -u "+%Y-%U")" >> "$GITHUB_OUTPUT"
4742

4843
- name: Setup ccache cache
4944
id: presto_cpp_ccache
5045
uses: actions/cache@v3
5146
with:
52-
path: ~/.ccache
53-
key: ${{ runner.os }}-presto-native-ccache-${{ steps.get-date.outputs.date }}
54-
restore-keys: |
55-
${{ runner.os }}-presto-native-ccache-
47+
path: ${GITHUB_WORKSPACE}/.ccache
48+
key: ${{ runner.os }}-presto-native-ccache-${{ steps.get-ccache-key.outputs.CCACHE_KEY }}
5649

5750
- name: Initiate ccache
5851
if: steps.presto_cpp_ccache.outputs.cache-hit != 'true'
5952
run: |
60-
mkdir -p ~/.ccache
61-
export CCACHE_DIR=$(realpath ~/.ccache)
62-
ccache -sz -M 5Gi
53+
rm -rf ${GITHUB_WORKSPACE}/.ccache
54+
mkdir -p ${GITHUB_WORKSPACE}/.ccache
55+
export CCACHE_DIR=${GITHUB_WORKSPACE}/.ccache
56+
ccache -sz -M 6Gi
6357
6458
- name: Build presto_cpp
6559
run: |
6660
source /opt/rh/gcc-toolset-9/enable
67-
export CCACHE_DIR=$(realpath ~/.ccache)
61+
export CCACHE_DIR=${GITHUB_WORKSPACE}/.ccache
6862
ccache -s
6963
cd ${GITHUB_WORKSPACE}/presto-native-execution
7064
make velox-submodule
@@ -81,27 +75,18 @@ jobs:
8175
run: |
8276
cd ${GITHUB_WORKSPACE}/presto-native-execution/_build/debug
8377
# Delete static libraries
84-
rm `find ./ -name lib\*.a`
85-
86-
- name: Cache local Maven repository
87-
id: cache-maven
88-
uses: actions/cache@v3
89-
with:
90-
path: ~/.m2/repository
91-
key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
92-
restore-keys: |
93-
${{ runner.os }}-maven-2-
94-
95-
- name: Populate maven cache
96-
if: steps.cache-maven.outputs.cache-hit != 'true'
97-
run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
78+
rm `find ./ -name lib\*.a`
9879
9980
- name: Maven Install
10081
run: |
10182
export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}"
102-
./mvnw install ${MAVEN_FAST_INSTALL} -pl '!presto-docs,!presto-server,!presto-server-rpm,!presto-test-coverage'
83+
# build all dependent components of presto-native-execution
84+
./mvnw clean install ${MAVEN_FAST_INSTALL} -pl 'presto-native-execution' -am
10385
10486
- name: Run e2e tests
10587
run: |
10688
rm -rf /tmp/hive_data/tpch/
107-
./mvnw test ${MAVEN_TEST} -pl 'presto-native-execution' -DPRESTO_SERVER=${GITHUB_WORKSPACE}/presto-native-execution/_build/debug/presto_cpp/main/presto_server -DDATA_DIR=/tmp/ -Duser.timezone=America/Bahia_Banderas
89+
90+
for test in $(find ${GITHUB_WORKSPACE}/presto-native-execution/src/test -name 'Test*.java' | grep -o "Test[a-zA-Z]*"); do
91+
./mvnw test ${MAVEN_TEST} -pl 'presto-native-execution' -Dtest=${test} -DPRESTO_SERVER=${GITHUB_WORKSPACE}/presto-native-execution/_build/debug/presto_cpp/main/presto_server -DDATA_DIR=/tmp/ -Duser.timezone=America/Bahia_Banderas
92+
done

‎presto-native-execution/src/test/java/com/facebook/presto/spark/TestPrestoSparkNativeGeneralQueries.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,4 +77,9 @@ public void testSubqueries() {}
7777
@Override
7878
@Ignore
7979
public void testTopN() {}
80+
81+
// This test is flaky: https://github.com/prestodb/presto/issues/19665
82+
@Override
83+
@Ignore
84+
public void testUnionAll() {}
8085
}

0 commit comments

Comments
 (0)