feat(trip-cluster): ingest — scopeable, incremental, processed read-back

2026-06-27 17:21:38 +02:00
parent 8d58ec03a9
commit 6d0c9662ee
3 changed files with 201 additions and 1 deletions
@@ -0,0 +1,104 @@
+import os
+from photoflow.core import Store
+from photoflow.immich import pipeline
+from app.ingest import run_ingest
+
+
+class FakeImmich:
+    def __init__(self, assets, tag_map=None, thumb=b"\xff\xd8\xffjpeg"):
+        self._assets = assets
+        self._tag_map = tag_map or {}
+        self._thumb = thumb
+        self.searches = []
+
+    def resolve_tag_id(self, name):
+        return self._tag_map.get(name)
+
+    def search_assets(self, **kwargs):
+        self.searches.append(kwargs)
+        return list(self._assets)
+
+    def download_thumbnail(self, asset_id):
+        return self._thumb
+
+
+def _asset(i, taken, tags=None, updated="2026-01-01T00:00:00Z"):
+    return {"id": i, "original_filename": f"{i}.jpg", "taken_at": taken,
+            "gps_lat": None, "gps_lon": None, "place_city": None,
+            "place_country": None, "type": "IMAGE", "tags": tags or [],
+            "rating": 0, "updated_at": updated}
+
+
+def test_ingest_upserts_assets_tags_and_thumbs(tmp_path):
+    store = Store(str(tmp_path / "t.db")).connect()
+    thumbs = str(tmp_path / "thumbs")
+    os.makedirs(thumbs, exist_ok=True)
+    client = FakeImmich([_asset("a", "2019-06-01", tags=["Italy 2019"]),
+                         _asset("b", "2019-06-02")])
+    res = run_ingest(client, store, thumbs, date_from="2019-01-01", date_to="2020-01-01")
+    assert res["fetched"] == 2
+    assert store.get_asset("a").taken_at == "2019-06-01"
+    assert store.asset_tags("a") == ["Italy 2019"]
+    assert os.path.exists(os.path.join(thumbs, "a.jpg"))
+    assert store.get_meta("last_ingest_at") == "2026-01-01T00:00:00Z"
+    store.close()
+
+
+def test_ingest_marks_processed_from_pipeline_tag(tmp_path):
+    store = Store(str(tmp_path / "t.db")).connect()
+    thumbs = str(tmp_path / "thumbs"); os.makedirs(thumbs, exist_ok=True)
+    client = FakeImmich([_asset("a", "2019-06-01", tags=[pipeline.PROCESSED])])
+    res = run_ingest(client, store, thumbs)
+    assert res["processed_marked"] == 1
+    assert store.get_asset("a").processed is True
+    store.close()
+
+
+def test_ingest_incremental_passes_updated_after(tmp_path):
+    store = Store(str(tmp_path / "t.db")).connect()
+    thumbs = str(tmp_path / "thumbs"); os.makedirs(thumbs, exist_ok=True)
+    store.set_meta("last_ingest_at", "2026-05-01T00:00:00Z")
+    client = FakeImmich([_asset("a", "2019-06-01")])
+    run_ingest(client, store, thumbs)
+    assert client.searches[0].get("updated_after") == "2026-05-01T00:00:00Z"
+    # --full ignores it
+    client2 = FakeImmich([_asset("a", "2019-06-01")])
+    run_ingest(client2, store, thumbs, full=True)
+    assert client2.searches[0].get("updated_after") is None
+    store.close()
+
+
+def test_ingest_subset_and_tag_resolution(tmp_path):
+    store = Store(str(tmp_path / "t.db")).connect()
+    thumbs = str(tmp_path / "thumbs"); os.makedirs(thumbs, exist_ok=True)
+    client = FakeImmich([_asset("a", "2019-06-01"), _asset("b", "2019-06-02")],
+                        tag_map={"Italy 2019": "t1"})
+    res = run_ingest(client, store, thumbs, tag="Italy 2019", subset=1)
+    assert res["fetched"] == 1
+    assert client.searches[0].get("tag_ids") == ["t1"]
+    store.close()
+
+
+def test_ingest_resilient_to_thumb_failure_and_zero_byte(tmp_path):
+    store = Store(str(tmp_path / "t.db")).connect()
+    thumbs = str(tmp_path / "thumbs"); os.makedirs(thumbs, exist_ok=True)
+
+    class FailThenOk(FakeImmich):
+        def download_thumbnail(self, asset_id):
+            if asset_id == "a":
+                raise RuntimeError("boom")
+            return self._thumb
+
+    # Pre-existing 0-byte file for "b" must be treated as missing and refetched.
+    with open(os.path.join(thumbs, "b.jpg"), "wb"):
+        pass
+
+    client = FailThenOk([_asset("a", "2019-06-01"), _asset("b", "2019-06-02")])
+    res = run_ingest(client, store, thumbs)
+    # The run completes despite "a" failing.
+    assert res["fetched"] == 2
+    # No 0-byte file left for the failed asset.
+    assert not os.path.exists(os.path.join(thumbs, "a.jpg"))
+    # The 0-byte file for "b" was replaced with real bytes.
+    assert os.path.getsize(os.path.join(thumbs, "b.jpg")) > 0
+    store.close()