Skip to content

Commit d2a1598

Browse files
author
dagz-mirror
committed
dagz: CI integration
1 parent ac9ca50 commit d2a1598

4 files changed

Lines changed: 249 additions & 0 deletions

File tree

‎.dagz/autoload.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
from dagz.integ.psycopg import PG_CONFIG
2+
from dagz.integ.pymysql import MYSQL_CONFIG
3+
4+
5+
def setup_adbc_driver():
6+
"""Custom ADBC driver rerouting using DAGZ rerouting framework."""
7+
import adbc_driver_postgresql.dbapi
8+
_orig_adbc_connect = adbc_driver_postgresql.dbapi.connect
9+
10+
def _override_connect(uri, *args, **kwargs):
11+
uri = PG_CONFIG.maybe_reroute_uri(uri)
12+
return _orig_adbc_connect(uri, *args, **kwargs)
13+
14+
adbc_driver_postgresql.dbapi.connect = _override_connect
15+
16+
17+
def setup_parallel_db():
18+
import dagz.integ.psycopg2
19+
import dagz.integ.pymysql
20+
21+
PG_CONFIG.configure(
22+
rewrite_db_name=PG_CONFIG.default_rewrite_db_name,
23+
should_reroute=PG_CONFIG.default_should_reroute,
24+
worker_init=dagz.integ.psycopg2.create_worker_init(["pandas"], host="127.0.0.1", port=5432, user="postgres", password="postgres"),
25+
prepare=None,
26+
)
27+
MYSQL_CONFIG.configure(
28+
rewrite_db_name=MYSQL_CONFIG.default_rewrite_db_name,
29+
should_reroute=MYSQL_CONFIG.default_should_reroute,
30+
worker_init=dagz.integ.pymysql.create_worker_init(["pandas"], host="127.0.0.1", port=3306, user="root", password=""),
31+
prepare=None,
32+
)
33+
34+
35+
def init(in_pytest):
36+
setup_parallel_db()
37+
setup_adbc_driver()

‎.dagz/config.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
project:
2+
uri: "git@github.com:pandas-dev/pandas.git"
3+
timezone: UTC
4+
public_url: "https://cloud.dagz.run"

‎.github/workflows/dagz-tests.yml‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
name: Dagz Tests
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
branches:
9+
- main
10+
workflow_dispatch:
11+
12+
permissions:
13+
contents: read
14+
15+
defaults:
16+
run:
17+
shell: bash -euo pipefail {0}
18+
19+
concurrency:
20+
group: dagz-tests-${{ github.event.pull_request.head.ref || github.ref }}
21+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
22+
23+
env:
24+
PIXI_ENVIRONMENT: py313
25+
DAGZ_URL: https://cloud.dagz.run
26+
DAGZ_PROFILE: py313
27+
28+
jobs:
29+
test:
30+
runs-on: ubuntu-24.04
31+
timeout-minutes: 120
32+
33+
services:
34+
mysql:
35+
image: mysql:9
36+
env:
37+
MYSQL_ALLOW_EMPTY_PASSWORD: yes
38+
MYSQL_DATABASE: pandas
39+
options: >-
40+
--health-cmd "mysqladmin ping"
41+
--health-interval 10s
42+
--health-timeout 5s
43+
--health-retries 5
44+
ports:
45+
- 3306:3306
46+
47+
postgres:
48+
image: postgres:18
49+
env:
50+
PGUSER: postgres
51+
POSTGRES_USER: postgres
52+
POSTGRES_PASSWORD: postgres
53+
POSTGRES_DB: pandas
54+
options: >-
55+
--health-cmd pg_isready
56+
--health-interval 10s
57+
--health-timeout 5s
58+
--health-retries 5
59+
ports:
60+
- 5432:5432
61+
62+
moto:
63+
image: motoserver/moto:5.1.18
64+
ports:
65+
- 5000:5000
66+
67+
steps:
68+
- name: Checkout
69+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
70+
with:
71+
fetch-depth: 0
72+
73+
- name: Resolve upstream commit
74+
env:
75+
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
76+
run: |
77+
if [ -n "$PR_HEAD_SHA" ]; then
78+
echo "GIT_COMMIT_HASH=$PR_HEAD_SHA" >> "$GITHUB_ENV"
79+
else
80+
echo "GIT_COMMIT_HASH=$(git rev-parse HEAD^1)" >> "$GITHUB_ENV"
81+
fi
82+
83+
- name: Unpack collector certificate
84+
env:
85+
DAGZ_COLLECTOR_BUNDLE: ${{ secrets.DAGZ_COLLECTOR_BUNDLE }}
86+
run: |
87+
mkdir -p "$RUNNER_TEMP/dagz-home/collector/tls"
88+
echo "$DAGZ_COLLECTOR_BUNDLE" | base64 -d | tar -x -C "$RUNNER_TEMP/dagz-home/collector/tls"
89+
echo "DAGZ_HOME=$RUNNER_TEMP/dagz-home" >> "$GITHUB_ENV"
90+
91+
- name: Create virtual environment with Pixi
92+
uses: ./.github/actions/setup-pixi
93+
with:
94+
environment: ${{ env.PIXI_ENVIRONMENT }}
95+
96+
- name: Build pandas
97+
run: |
98+
pixi run --environment "$PIXI_ENVIRONMENT" build-pandas --editable
99+
100+
- name: Install dagz
101+
run: |
102+
pixi run --environment "$PIXI_ENVIRONMENT" python -m pip install --pre dagz
103+
104+
- name: Test
105+
env:
106+
CI_NODE_TOTAL: 1
107+
CI_NODE_INDEX: 1
108+
PANDAS_FUTURE_INFER_STRING: 1
109+
PANDAS_FUTURE_PYTHON_SCALARS: 0
110+
PANDAS_MOTO_URL: http://localhost:5000
111+
QT_QPA_PLATFORM: offscreen
112+
run: |
113+
pixi run --environment "$PIXI_ENVIRONMENT" \
114+
python -m pytest -r fE --dagz --dagz-workers=4 -m "not single_cpu" pandas

‎.github/workflows/mirror-upstream.yml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: Mirror Upstream
2+
3+
on:
4+
schedule:
5+
- cron: "*/5 * * * *"
6+
workflow_dispatch:
7+
inputs:
8+
max_prs:
9+
description: Maximum PR branches to update in this run
10+
default: "2"
11+
12+
permissions:
13+
contents: read
14+
pull-requests: write
15+
16+
defaults:
17+
run:
18+
shell: bash -euo pipefail {0}
19+
20+
concurrency:
21+
group: mirror-upstream
22+
cancel-in-progress: false
23+
24+
env:
25+
UPSTREAM: https://github.com/pandas-dev/pandas.git
26+
MAX_PRS: ${{ inputs.max_prs || '2' }}
27+
28+
jobs:
29+
mirror:
30+
runs-on: ubuntu-24.04
31+
timeout-minutes: 30
32+
33+
steps:
34+
- name: Checkout
35+
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
36+
with:
37+
fetch-depth: 0
38+
token: ${{ secrets.MIRROR_TOKEN }}
39+
40+
- name: Mirror upstream refs
41+
env:
42+
GH_TOKEN: ${{ secrets.MIRROR_TOKEN }}
43+
run: |
44+
git config user.name "dagz-mirror"
45+
git config user.email "dagz-mirror@autodag.dev"
46+
git remote add upstream "$UPSTREAM"
47+
48+
git fetch upstream main
49+
upstream_main=$(git rev-parse FETCH_HEAD)
50+
51+
if [ "$(git rev-parse origin/main^1)" != "$upstream_main" ]; then
52+
echo "main: rebasing dagz commit onto $upstream_main"
53+
git checkout -q -B main origin/main
54+
git rebase --onto "$upstream_main" main^ main
55+
git push --force origin main
56+
fi
57+
58+
mirrored_prs=$(gh api --paginate "repos/$GITHUB_REPOSITORY/pulls?state=open&per_page=100" --jq '.[].head.ref')
59+
60+
pushed=0
61+
pr_failures=0
62+
while IFS=$'\t' read -r number head_sha head_owner head_ref title; do
63+
branch="$head_owner/$head_ref"
64+
our_sha=$(git rev-parse --verify --quiet "origin/$branch" || true)
65+
66+
if [ "$our_sha" != "$head_sha" ]; then
67+
if [ "$pushed" -ge "$MAX_PRS" ]; then
68+
echo "reached MAX_PRS=$MAX_PRS, remaining PRs deferred to the next run"
69+
continue
70+
fi
71+
echo "$branch: $head_sha"
72+
git fetch upstream "refs/pull/$number/head"
73+
git push --force origin "$head_sha:refs/heads/$branch"
74+
pushed=$((pushed + 1))
75+
our_sha="$head_sha"
76+
fi
77+
78+
if [ -n "$our_sha" ] && ! grep -qxF "$branch" <<<"$mirrored_prs"; then
79+
if ! gh api --method POST "repos/$GITHUB_REPOSITORY/pulls" \
80+
-f "title=$title" \
81+
-f "head=$branch" \
82+
-f "base=main" \
83+
-f "body=Mirror of pandas-dev/pandas#$number. Runs the pandas test suite through dagz." \
84+
--jq '.html_url'; then
85+
pr_failures=$((pr_failures + 1))
86+
fi
87+
fi
88+
done < <(gh api "repos/pandas-dev/pandas/pulls?state=open&sort=updated&direction=desc&per_page=100" \
89+
--jq '.[] | [.number, .head.sha, .head.repo.owner.login, .head.ref, .title] | @tsv')
90+
91+
if [ "$pr_failures" -gt 0 ]; then
92+
echo "$pr_failures pull requests could not be created; mirrored branches are up to date"
93+
exit 1
94+
fi

0 commit comments

Comments
 (0)