Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ def migrate(source_db_url: str, dest_db_path: str) -> None:
# Migrate events
logger.info("Migrating events...")
events = source_session.query(v0_schema.StorageEvent).all()
migrated_events = 0
skipped_event_ids = []
for storage_event in events:
try:
event_obj = storage_event.to_event()
Expand All @@ -143,9 +145,22 @@ def migrate(source_db_url: str, dest_db_path: str) -> None:
event_data,
),
)
migrated_events += 1
except Exception as e:
logger.warning(f"Failed to migrate event {storage_event.id}: {e}")
logger.info(f"Migrated {len(events)} events.")
skipped_event_ids.append(storage_event.id)
logger.info(f"Migrated {migrated_events} events.")
if skipped_event_ids:
# The rows are dropped from the destination, so name them: the count above
# is the number that survived, not the number the source held.
logger.warning(
"Skipped %d event(s) that could not be migrated: %s. They are still"
" in the source database; once the cause is fixed, re-run the"
" migration into a new, empty destination (the session rows are"
" already in this one).",
len(skipped_event_ids),
", ".join(skipped_event_ids),
)

dest_conn.commit()
logger.info("Migration completed successfully.")
Expand Down
91 changes: 91 additions & 0 deletions tests/unittests/sessions/migration/test_migration.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import logging
import os
import pickle
import sqlite3
import time
from unittest import mock

Expand Down Expand Up @@ -874,3 +875,93 @@ def test_v1_storage_session_update_timestamp_tz() -> None:

def test_v0_storage_session_update_timestamp_tz() -> None:
_assert_update_timestamp_tz_is_utc_timestamp(v0)


def test_sqlite_migration_reports_only_migrated_events(
caplog, tmp_path
) -> None:
"""A row that fails to convert is skipped, so the count must not claim it."""
source_db_path = tmp_path / "source_sqlite.db"
dest_db_path = tmp_path / "dest_sqlite.db"
source_db_url = f"sqlite:///{source_db_path}"

source_engine = create_engine(source_db_url)
v0.Base.metadata.create_all(source_engine)
SourceSession = sessionmaker(bind=source_engine)
source_session = SourceSession()

now = datetime.now(timezone.utc)
source_session.add(
v0.StorageSession(
app_name="app1",
user_id="user1",
id="session1",
state={},
create_time=now,
update_time=now,
)
)
for event_id, invocation_id in (("event1", "invoke1"), ("event2", "invoke2")):
source_session.add(
v0.StorageEvent(
id=event_id,
app_name="app1",
user_id="user1",
session_id="session1",
invocation_id=invocation_id,
author="user",
actions=EventActions(state_delta={}),
timestamp=now,
)
)
source_session.commit()
source_session.close()

original_to_event = v0.StorageEvent.to_event

def to_event(self):
if self.id == "event2":
raise ValueError("unreadable event payload")
return original_to_event(self)

with mock.patch.object(v0.StorageEvent, "to_event", to_event):
with caplog.at_level(logging.INFO):
mfss.migrate(source_db_url, str(dest_db_path))

# Only the convertible row lands in the destination.
dest_conn = sqlite3.connect(dest_db_path)
try:
migrated_ids = [
row[0] for row in dest_conn.execute("SELECT id FROM events").fetchall()
]
finally:
dest_conn.close()
assert migrated_ids == ["event1"]

# The summary counts migrated rows, not source rows, and names the skipped one.
assert "Migrated 1 events." in caplog.text
assert "Migrated 2 events." not in caplog.text
assert "event2" in caplog.text
# Re-running into the same destination fails on the sessions UNIQUE
# constraint, so the advice has to point at a fresh one.
assert "new, empty destination" in caplog.text

# And that advice is what actually works: with the cause fixed, a second run
# into a new destination gets both rows.
caplog.clear()
dest_db_path2 = tmp_path / "dest_sqlite2.db"
with caplog.at_level(logging.INFO):
mfss.migrate(source_db_url, str(dest_db_path2))
dest_conn2 = sqlite3.connect(dest_db_path2)
try:
migrated_ids2 = [
row[0] for row in dest_conn2.execute("SELECT id FROM events").fetchall()
]
finally:
dest_conn2.close()
assert sorted(migrated_ids2) == ["event1", "event2"]
assert "Skipped" not in caplog.text

# The same destination cannot be reused, which is why the warning says so.
with pytest.raises(SystemExit):
mfss.migrate(source_db_url, str(dest_db_path2))
Loading