Skip to content

Commit b0c43e0

Browse files
abrignoniclaude
andcommitted
Store Python's text for floats and booleans in LAVA and the KML sidecar
A float or boolean bound into a TEXT column is turned into text by SQLite, and that text depends on the SQLite version Python links. Through 3.51 a float is rounded to 15 significant digits, so 35.65956623101914 was stored as 35.6595662310191. From 3.52.0 it is rounded to 17, so 670162.147879839 was stored as 670162.14787983894. A boolean was stored as 1 or 0. The same image gave different LAVA values on different machines, and the list path test failed on any Python linking SQLite 3.52.0 or later. The list path now stores the text Python prints, as the streamed path already did, so LAVA matches the HTML, TSV, timeline and KML output on every SQLite version. kmlgen stores the coordinates in _KML Exports/_latlong.db the same way. A date in an untyped streamed column is now bound as text too, so it no longer goes through sqlite3's deprecated default adapters. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent d5afb4c commit b0c43e0

4 files changed

Lines changed: 66 additions & 34 deletions

File tree

‎admin/test/scripts/test_artifact_result.py‎

Lines changed: 46 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,15 @@
1212
import sys
1313
import tempfile
1414
import unittest
15+
import warnings
1516

1617
REPO_ROOT = pathlib.Path(__file__).resolve().parents[3]
1718
sys.path.insert(0, str(REPO_ROOT))
1819

1920
from scripts import lavafuncs # pylint: disable=wrong-import-position
2021
from scripts.context import Context # pylint: disable=wrong-import-position
2122
from leapp_functions.app.artifact_result import ArtifactResult # pylint: disable=wrong-import-position
22-
from scripts.ilapfuncs import artifact_processor, OutputParameters # pylint: disable=wrong-import-position
23+
from scripts.ilapfuncs import artifact_processor, kmlgen, OutputParameters # pylint: disable=wrong-import-position
2324

2425
UTC = datetime.timezone.utc
2526
STAMP = datetime.datetime(2022, 3, 4, 12, 34, 56, tzinfo=UTC)
@@ -281,8 +282,10 @@ def test_list_module_that_raises_leaves_the_same_nothing(self):
281282
self.assertNotIn('Probe Category', lavafuncs.lava_data['artifacts'])
282283

283284
class TestReplayKeepsPythonText(ArtifactResultTestCase):
284-
"""Floats and booleans in untyped columns replay exactly as the list path prints them,
285-
while the list path's own stored values are unchanged."""
285+
"""Floats and booleans in untyped LAVA columns, and the coordinates kmlgen keeps in
286+
_latlong.db, are stored as the text Python prints for them. The stored text is then the
287+
same whatever SQLite library the run links, matches the HTML, TSV and KML output, and
288+
replays as the original values."""
286289

287290
def setUp(self):
288291
super().setUp()
@@ -299,24 +302,53 @@ def test_streamed_rows_replay_as_the_original_values(self):
299302
replayed = list(lavafuncs.lava_iter_artifact_rows(
300303
result.table_name, headers, result.object_columns, result.row_count))
301304
self.assertEqual(replayed, rows)
302-
# The stored text is Python's own rendering, not SQLite's 15-digit one.
305+
# The stored text is Python's own rendering, not SQLite's, which depends on its version.
303306
stored = lavafuncs.lava_db.execute(
304307
f'SELECT latitude, flag FROM {lavafuncs.quote_sql_name(result.table_name)} ORDER BY rowid').fetchall()
305308
self.assertEqual(stored, [('35.65956623101914', 'True'), ('-78.87282251025944', 'False')])
306309

307-
def test_list_path_storage_is_unchanged(self):
308-
headers = ('Latitude', 'Flag', 'When')
310+
def test_list_path_stores_python_text(self):
311+
headers = ('Latitude', 'Flag', ('When', 'datetime'))
309312
table, object_columns, column_map = lavafuncs.lava_process_artifact(
310-
'Unit Category', 'unit_module', 'List Artifact', (headers[0], headers[1], (headers[2], 'datetime')),
311-
1, func_name='list_artifact')
313+
'Unit Category', 'unit_module', 'List Artifact', headers, 2, func_name='list_artifact')
312314
lavafuncs.lava_insert_sqlite_data(
313-
table, [(35.65956623101914, True, '2024-07-21T15:16:47.357843+00:00')],
314-
object_columns, (headers[0], headers[1], (headers[2], 'datetime')), column_map)
315+
table, [(35.65956623101914, True, '2024-07-21T15:16:47.357843+00:00'),
316+
(670162.147879839, False, '2024-07-21T15:16:47.357843+00:00')],
317+
object_columns, headers, column_map)
315318
stored = lavafuncs.lava_db.execute(
316-
f'SELECT latitude, flag, "when" FROM {lavafuncs.quote_sql_name(table)}').fetchone()
317-
# SQLite's rendering of a bound float and boolean into TEXT columns, and a whole-second
318-
# int epoch for an ISO string: what the list path stored before this change.
319-
self.assertEqual(stored, ('35.6595662310191', '1', 1721575007))
319+
f'SELECT latitude, flag, "when" FROM {lavafuncs.quote_sql_name(table)} ORDER BY rowid').fetchall()
320+
# Bound as numbers, SQLite 3.51 and earlier store the first float as 35.6595662310191,
321+
# 3.52.0 and later store the second as 670162.14787983894, and both store a boolean as
322+
# 1 or 0. The datetime column still stores the whole-second epoch.
323+
self.assertEqual(stored, [('35.65956623101914', 'True', 1721575007),
324+
('670162.147879839', 'False', 1721575007)])
325+
326+
def test_streamed_dates_in_an_untyped_column_are_bound_as_text(self):
327+
# As on the list path, sqlite3's default date adapters, deprecated from Python 3.12,
328+
# never see a value, and the stored text is what they wrote.
329+
with warnings.catch_warnings(record=True) as caught:
330+
warnings.simplefilter('always')
331+
result = Context.create_artifact_result(headers=HEADERS)
332+
for row in ROWS:
333+
result.add_row(row)
334+
result.close()
335+
self.assertEqual([w for w in caught if issubclass(w.category, DeprecationWarning)
336+
and 'adapter' in str(w.message)], [])
337+
stored = [row[0] for row in lavafuncs.lava_db.execute(
338+
f'SELECT moment FROM {lavafuncs.quote_sql_name(result.table_name)} ORDER BY rowid')]
339+
self.assertEqual(stored, MOMENT_TEXT)
340+
341+
def test_kml_sidecar_stores_python_text_for_coordinates(self):
342+
report_folder = pathlib.Path(self.tmpdir, 'category', 'artifact')
343+
report_folder.mkdir(parents=True)
344+
kmlgen(str(report_folder), 'Python Text', [(STAMP, 35.65956623101914, -78.87282251025944)],
345+
['Timestamp', 'Latitude', 'Longitude'])
346+
db = sqlite3.connect(pathlib.Path(self.tmpdir, '_KML Exports', '_latlong.db'))
347+
stored = db.execute('SELECT latitude, longitude FROM data').fetchall()
348+
db.close()
349+
# Bound as numbers these read 35.6595662310191 and -78.8728225102594 through
350+
# SQLite 3.51, and 35.659566231019142 and -78.872822510259439 from 3.52.0.
351+
self.assertEqual(stored, [('35.65956623101914', '-78.87282251025944')])
320352

321353
def test_python_text_restore_is_exact_and_leaves_other_text_alone(self):
322354
restore = lavafuncs._restore_lava_value # pylint: disable=protected-access

‎leapp_functions/app/artifact_result.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,6 @@ def _flush_batch(self):
171171
batch_size=self.batch_size,
172172
async_write=self.async_write,
173173
queue_size=self.queue_size,
174-
keep_python_text=True,
175174
)
176175
self.row_count += inserted_count
177176

‎scripts/ilapfuncs.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565
from scripts.lavafuncs import lava_process_artifact, lava_insert_sqlite_data, lava_iter_artifact_rows, \
6666
lava_get_media_item, \
6767
lava_insert_sqlite_media_item, lava_insert_sqlite_media_references, lava_get_media_references, \
68-
lava_get_full_media_info, lava_update_record_count, bind_dates_as_text
68+
lava_get_full_media_info, lava_update_record_count, bind_dates_as_text, _python_text
6969

7070
os.path.basename = lru_cache(maxsize=None)(os.path.basename)
7171

@@ -1318,7 +1318,9 @@ def kmlgen(report_folder, kmlactivity, data_list, data_headers):
13181318
pnt.name = times
13191319
pnt.description = f"{times_header}: {times} - {kmlactivity}"
13201320
pnt.coords = [(lon, lat)]
1321-
data.append((bind_dates_as_text(times), lat, lon, kmlactivity))
1321+
# Python's text for the coordinates, as the .kml file carries: bound as floats,
1322+
# the text SQLite stores depends on its version (see lavafuncs._python_text).
1323+
data.append((bind_dates_as_text(times), _python_text(lat), _python_text(lon), kmlactivity))
13221324

13231325
if len(data) > 0:
13241326
report_folder = report_folder.rstrip('/')

‎scripts/lavafuncs.py‎

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -179,12 +179,14 @@ def _python_text(value):
179179
"""
180180
The text Python itself would print for a float or a boolean, or the value unchanged.
181181
182-
An untyped LAVA column is TEXT, and SQLite renders a bound float into it at 15
183-
significant digits and a boolean as 1 or 0. The HTML, TSV, timeline and KML writers
184-
print the Python value, so rows replayed out of LAVA would otherwise read
185-
35.6595662310191 where the list path wrote 35.65956623101914, and 1 where it wrote
186-
True. Storing Python's own text keeps the replayed outputs identical to the list
187-
path's; _restore_lava_value turns it back into the value.
182+
An untyped LAVA column is TEXT, so a float or boolean bound as a number is turned into
183+
text by SQLite, and that text depends on the SQLite library the run links. Through
184+
3.51 a float is rounded to 15 significant digits, so 35.65956623101914 is stored as
185+
35.6595662310191; 3.52.0 and later round to 17 (https://sqlite.org/releaselog/3_53_0.html),
186+
so 670162.147879839 is stored as 670162.14787983894. A boolean is stored as 1 or 0.
187+
The HTML, TSV, timeline and KML writers print the Python value, so storing Python's own
188+
text keeps LAVA identical to them on every SQLite version; _restore_lava_value turns it
189+
back into the value when streamed rows are replayed.
188190
"""
189191

190192
if isinstance(value, bool):
@@ -194,24 +196,23 @@ def _python_text(value):
194196
return value
195197

196198

197-
def _prepare_lava_value(value, column_type=None, keep_python_text=False):
199+
def _prepare_lava_value(value, column_type=None):
198200
"""
199201
Convert a Python value into the representation stored in the LAVA SQLite table.
200202
201203
Only column types that need storage conversion get handlers here. Other object
202204
column types, such as phonenumber, remain normal SQLite TEXT values while their
203205
type metadata stays available in the LAVA artifact metadata.
204206
205-
keep_python_text stores a float or boolean in an untyped column as the text Python
206-
prints for it (see _python_text). Rows that will be replayed for the other outputs
207-
ask for it; the list path does not, so its stored values are unchanged.
207+
A float or boolean in an untyped column is stored as the text Python prints for it
208+
(see _python_text), whether the artifact returned a list or streamed its rows.
208209
"""
209210

210211
if isinstance(value, (dict, list)):
211212
return json.dumps(value)
212213

213-
if keep_python_text and column_type is None:
214-
return _python_text(value)
214+
if column_type is None:
215+
value = _python_text(value)
215216

216217
type_handlers = {
217218
'datetime': _prepare_datetime_value,
@@ -626,12 +627,12 @@ def lava_insert_sqlite_data(
626627
column_map, # pylint: disable=unused-argument
627628
batch_size=10000,
628629
async_write=False,
629-
queue_size=5000,
630-
keep_python_text=False):
630+
queue_size=5000):
631631
"""
632632
Insert data into a SQLite database table with automatic column sanitization and type conversion.
633633
This function handles the insertion of multiple rows of data into a specified SQLite table,
634634
with special handling for complex data types (dict, list) and object-column type conversions.
635+
Floats and booleans in untyped columns are stored as the text Python prints for them.
635636
Args:
636637
table_name (str): The name of the SQLite table to insert data into.
637638
data (iterable): Rows to insert, where each row is a sequence of values
@@ -645,8 +646,6 @@ def lava_insert_sqlite_data(
645646
batch_size (int): Maximum rows to insert per SQLite executemany call.
646647
async_write (bool): If True, prepare and insert rows on a writer thread.
647648
queue_size (int): Maximum rows waiting for the async writer.
648-
keep_python_text (bool): Store floats and booleans in untyped columns as the text
649-
Python prints for them, for rows that will be replayed.
650649
Returns:
651650
int: Number of rows inserted.
652651
"""
@@ -670,7 +669,7 @@ def prepare_row(row):
670669
row = tuple(row)
671670
processed_row = []
672671
for index, value in enumerate(row):
673-
processed_row.append(_prepare_lava_value(value, column_types[index], keep_python_text))
672+
processed_row.append(_prepare_lava_value(value, column_types[index]))
674673
return tuple(processed_row)
675674

676675
if async_write:

0 commit comments

Comments
 (0)