Skip to content
This repository was archived by the owner on May 17, 2024. It is now read-only.

cli: output diff as jsonl, stats as json #90

Merged
sirupsen merged 3 commits into from
Jun 22, 2022
Merged

cli: output diff as jsonl, stats as json #90

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f09a866
cli: output diff as jsonl, stats as json
sirupsen Jun 21, 2022
94d1419
cli: add --json for stats, table1 + table2 counts
sirupsen Jun 21, 2022
37b47a0
cli: only json from standard diff with --json
sirupsen Jun 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
cli: add --json for stats, table1 + table2 counts
  • Loading branch information
@sirupsen
sirupsen committed Jun 21, 2022
commit 94d1419091ee4c6266a4ea66713ff8ee280d9376
29 changes: 17 additions & 12 deletions data_diff/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
@click.option("--max-age", default=None, help="Considers only rows younger than specified. See --min-age.")
@click.option("-s", "--stats", is_flag=True, help="Print stats instead of a detailed diff")
@click.option("-d", "--debug", is_flag=True, help="Print debug info")
@click.option("--json", 'json_output', is_flag=True, help="Print JSON output for --stats")
@click.option("-v", "--verbose", is_flag=True, help="Print extra info")
@click.option("-i", "--interactive", is_flag=True, help="Confirm queries, implies --debug")
@click.option("--keep-column-case", is_flag=True, help="Don't use the schema to fix the case of given column names.")
Expand Down Expand Up @@ -81,6 +82,7 @@ def main(
interactive,
threads,
keep_column_case,
json_output,
):
if limit and stats:
print("Error: cannot specify a limit when using the -s/--stats switch")
Expand Down Expand Up @@ -145,21 +147,24 @@ def main(
if stats:
diff = list(diff_iter)
unique_diff_count = len({i[0] for _, i in diff})
table1_count = differ.stats.get("table1_count")
percent = 100 * unique_diff_count / (table1_count or 1)
max_table_count = max(differ.stats["table1_count"], differ.stats["table2_count"])
percent = 100 * unique_diff_count / (max_table_count or 1)
plus = len([1 for op, _ in diff if op == "+"])
minus = len([1 for op, _ in diff if op == "-"])

count = differ.stats["table_count"]
diff = {
"different_rows": len(diff),
"different_percent": percent,
"different_+": plus,
"different_-": minus,
"total": count,
}

print(json.dumps(diff, indent=2))
if json_output:
json_output = {
"different_rows": len(diff),
"different_percent": percent,
"different_+": plus,
"different_-": minus,
"total": max_table_count,
}
print(json.dumps(json_output, indent=2))
else:
print(f"Diff-Total: {len(diff)} changed rows out of {max_table_count}")
print(f"Diff-Percent: {percent:.14f}%")

@sirupsen sirupsen Jun 21, 2022

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

BTW I stopped rounding aggressively here because e.g. when I have 1 different row for 100m entries, it just shows 0 with that previous rounding

print(f"Diff-Split: +{plus} -{minus}")
else:
for op, key in diff_iter:
color = COLOR_SCHEME[op]
Expand Down
8 changes: 5 additions & 3 deletions data_diff/diff_tables.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,8 +384,9 @@ def _bisect_and_diff_tables(self, table1, table2, level=0, max_rows=None):

# This happens when the initial bisection threshold is larger than
# the table itself.
if level == 0 and not self.stats.get("table_count", False):
self.stats["table_count"] = self.stats.get("table_count", 0) + max(len(rows1), len(rows2))
if level == 0 and not self.stats.get("table1_count", False):

@erezsh erezsh Jun 22, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch. But I think we always want to override these attribute. Why would we care what previous values they held?

@sirupsen sirupsen Jun 22, 2022

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(talked about on slack, just for the record) Will double count if it's not level 0, but we can remove the not

self.stats["table1_count"] = self.stats.get("table1_count", 0) + len(rows1)
self.stats["table2_count"] = self.stats.get("table2_count", 0) + len(rows2)

@erezsh erezsh Jun 22, 2022

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Still this line... just

self.stats["table1_count"] = len(rows1)

No?


logger.info(". " * level + f"Diff found {len(diff)} different rows.")
self.stats["rows_downloaded"] = self.stats.get("rows_downloaded", 0) + max(len(rows1), len(rows2))
Expand Down Expand Up @@ -426,7 +427,8 @@ def _diff_tables(self, table1, table2, level=0, segment_index=None, segment_coun
return

if level == 1:
self.stats["table_count"] = self.stats.get("table_count", 0) + max(count1, count2)
self.stats["table1_count"] = self.stats.get("table_count1", 0) + count1
self.stats["table2_count"] = self.stats.get("table_count2", 0) + count2

if checksum1 != checksum2:
yield from self._bisect_and_diff_tables(table1, table2, level=level, max_rows=max(count1, count2))
Expand Down