feat(trip-cluster): ingest — scopeable, incremental, processed read-back

2026-06-27 17:21:38 +02:00
parent 8d58ec03a9
commit 6d0c9662ee
3 changed files with 201 additions and 1 deletions
@@ -18,6 +18,19 @@ def _immich(deps, cfg):
    return ImmichClient(cfg.immich_url, cfg.immich_api_key)


+def cmd_ingest(deps, *, date_from, date_to, tag, subset, full) -> int:
+    cfg = deps["config"]
+    store = _store(cfg)
+    client = _immich(deps, cfg)
+    from app.ingest import run_ingest
+    res = run_ingest(client, store, cfg.thumbs_dir, date_from=date_from,
+                     date_to=date_to, tag=tag, subset=subset, full=full)
+    store.close()
+    print(f"Ingested {res['fetched']} asset(s); "
+          f"{res['processed_marked']} already-processed.")
+    return 0
+
+
 def cmd_serve(deps) -> int:
    from app import create_app
    create_app(deps["config"]).run(host="0.0.0.0", port=8084)
@@ -51,7 +64,10 @@ def main(argv=None) -> int:
    deps = {"config": load_config()}
    if args.command == "serve":
        return cmd_serve(deps)
-    # ingest / cluster / apply are wired in later tasks.
+    if args.command == "ingest":
+        return cmd_ingest(deps, date_from=args.date_from, date_to=args.date_to,
+                          tag=args.tag, subset=args.subset, full=args.full)
+    # cluster / apply are wired in later tasks.
    print(f"Command '{args.command}' is not implemented yet.")
    return 1

@@ -0,0 +1,80 @@
+import datetime
+import logging
+import os
+
+from photoflow.core.models import Asset
+from photoflow.immich import pipeline
+
+log = logging.getLogger(__name__)
+
+
+def _now() -> str:
+    return datetime.datetime.now(datetime.timezone.utc).isoformat()
+
+
+def run_ingest(client, store, thumbs_dir, *, date_from=None, date_to=None,
+               tag=None, subset=None, full=False) -> dict:
+    os.makedirs(thumbs_dir, exist_ok=True)
+
+    tag_ids = None
+    if tag:
+        tid = client.resolve_tag_id(tag)
+        if not tid:
+            raise ValueError(f"Tag {tag!r} not found in Immich")
+        tag_ids = [tid]
+
+    updated_after = None if full else store.get_meta("last_ingest_at")
+
+    assets = client.search_assets(taken_after=date_from, taken_before=date_to,
+                                  tag_ids=tag_ids, updated_after=updated_after)
+    if subset is not None:
+        assets = assets[:subset]
+
+    processed_marked = 0
+    max_updated = updated_after or ""
+    tag_counts: dict = {}
+
+    for a in assets:
+        # Revision (2): sanitize the Immich-provided id before using it as a
+        # path component, guarding against path traversal (mirror /thumb proxy).
+        safe_id = os.path.basename(a["id"])
+        thumb_path = os.path.join(thumbs_dir, f"{safe_id}.jpg")
+        # Revision (1): treat an existing 0-byte file as missing.
+        need_thumb = (not os.path.exists(thumb_path)
+                      or os.path.getsize(thumb_path) == 0)
+        if need_thumb:
+            try:
+                data = client.download_thumbnail(a["id"])
+                # Write to a temp path and os.replace into place so a failure
+                # never leaves a 0-byte <id>.jpg behind.
+                tmp_path = f"{thumb_path}.tmp"
+                with open(tmp_path, "wb") as f:
+                    f.write(data)
+                os.replace(tmp_path, thumb_path)
+            except Exception:
+                log.exception("Thumbnail download failed for asset %s", a["id"])
+                continue
+
+        store.upsert_asset(Asset(
+            immich_id=a["id"], taken_at=a["taken_at"], gps_lat=a["gps_lat"],
+            gps_lon=a["gps_lon"], place_city=a["place_city"],
+            place_country=a["place_country"], type=a["type"],
+            thumb_path=thumb_path, ingested_at=_now(), updated_at=a["updated_at"]))
+        store.set_asset_tags(a["id"], a["tags"])
+        for t in a["tags"]:
+            tag_counts[t] = tag_counts.get(t, 0) + 1
+
+        if pipeline.PROCESSED in a["tags"]:
+            store.mark_processed(a["id"])
+            processed_marked += 1
+
+        if a["updated_at"] and a["updated_at"] > max_updated:
+            max_updated = a["updated_at"]
+
+    for name, count in tag_counts.items():
+        store.upsert_tag(name, count=count)
+
+    if max_updated:
+        store.set_meta("last_ingest_at", max_updated)
+
+    return {"fetched": len(assets), "processed_marked": processed_marked}