fix(ingest): don't advance incremental cursor past a failed/sampled asset
A later successful asset could push last_ingest_at past an earlier asset whose thumbnail download failed, permanently excluding it from later incremental runs (only --full recovered it). Now the cursor never advances to/past the earliest failed asset, never below the prior cursor, and not at all on a --subset run. Regression tests added. (review finding #9)
This commit is contained in:
@@ -102,3 +102,39 @@ def test_ingest_resilient_to_thumb_failure_and_zero_byte(tmp_path):
|
||||
# The 0-byte file for "b" was replaced with real bytes.
|
||||
assert os.path.getsize(os.path.join(thumbs, "b.jpg")) > 0
|
||||
store.close()
|
||||
|
||||
|
||||
def test_ingest_watermark_not_advanced_past_failed_asset(tmp_path):
|
||||
# A later successful asset must NOT advance last_ingest_at past an earlier
|
||||
# asset whose thumbnail failed, or that asset is stranded on the next
|
||||
# incremental run (review finding #9).
|
||||
store = Store(str(tmp_path / "t.db")).connect()
|
||||
thumbs = str(tmp_path / "thumbs"); os.makedirs(thumbs, exist_ok=True)
|
||||
|
||||
class FailOld(FakeImmich):
|
||||
def download_thumbnail(self, asset_id):
|
||||
if asset_id == "old":
|
||||
raise RuntimeError("boom")
|
||||
return self._thumb
|
||||
|
||||
client = FailOld([_asset("old", "2008-06-01", updated="2026-01-01T00:00:00Z"),
|
||||
_asset("new", "2024-03-01", updated="2026-03-01T00:00:00Z")])
|
||||
run_ingest(client, store, thumbs)
|
||||
assert store.get_asset("old") is None and store.get_asset("new") is not None
|
||||
wm = store.get_meta("last_ingest_at")
|
||||
# cursor must stay below the failed asset (here: not advanced at all)
|
||||
assert wm is None or wm < "2026-01-01T00:00:00Z"
|
||||
assert wm != "2026-03-01T00:00:00Z"
|
||||
store.close()
|
||||
|
||||
|
||||
def test_ingest_subset_run_does_not_advance_watermark(tmp_path):
|
||||
# --subset is a sampling run; it must not move the incremental cursor or it
|
||||
# would strand the un-fetched remainder.
|
||||
store = Store(str(tmp_path / "t.db")).connect()
|
||||
thumbs = str(tmp_path / "thumbs"); os.makedirs(thumbs, exist_ok=True)
|
||||
client = FakeImmich([_asset("a", "2019-06-01", updated="2026-02-01T00:00:00Z"),
|
||||
_asset("b", "2019-06-02", updated="2026-02-02T00:00:00Z")])
|
||||
run_ingest(client, store, thumbs, subset=1)
|
||||
assert store.get_meta("last_ingest_at") is None
|
||||
store.close()
|
||||
|
||||
Reference in New Issue
Block a user