From 01ac95059e4429bf382a31eda02c634adfb83917 Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:08:11 +0200 Subject: [PATCH 01/19] =?UTF-8?q?feat:=20monorepo=20scaffold=20=E2=80=94?= =?UTF-8?q?=20photoflow=20shared=20dist=20+=20trip-cluster=20app=20skeleto?= =?UTF-8?q?n?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 6 ++++++ .gitignore | 8 ++++++++ apps/trip-cluster/app/__init__.py | 0 apps/trip-cluster/categorize.py | 5 +++++ apps/trip-cluster/pyproject.toml | 13 +++++++++++++ apps/trip-cluster/tests/conftest.py | 1 + pyproject.toml | 3 +++ shared/photoflow/__init__.py | 0 shared/photoflow/core/__init__.py | 0 shared/photoflow/immich/__init__.py | 0 shared/photoflow/ui/__init__.py | 0 shared/pyproject.toml | 16 ++++++++++++++++ shared/tests/test_imports.py | 4 ++++ 13 files changed, 56 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 apps/trip-cluster/app/__init__.py create mode 100644 apps/trip-cluster/categorize.py create mode 100644 apps/trip-cluster/pyproject.toml create mode 100644 apps/trip-cluster/tests/conftest.py create mode 100644 pyproject.toml create mode 100644 shared/photoflow/__init__.py create mode 100644 shared/photoflow/core/__init__.py create mode 100644 shared/photoflow/immich/__init__.py create mode 100644 shared/photoflow/ui/__init__.py create mode 100644 shared/pyproject.toml create mode 100644 shared/tests/test_imports.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..65fd3b9 --- /dev/null +++ b/.env.example @@ -0,0 +1,6 @@ +IMMICH_URL=http://your-immich-host:2283 +IMMICH_API_KEY=your-immich-api-key +ANTHROPIC_API_KEY= +DATA_DIR=./data +UID=1000 +GID=1000 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c92ae54 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +__pycache__/ +*.pyc +.venv/ +data/ +*.db +.env +.pytest_cache/ +*.egg-info/ diff --git a/apps/trip-cluster/app/__init__.py b/apps/trip-cluster/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/trip-cluster/categorize.py b/apps/trip-cluster/categorize.py new file mode 100644 index 0000000..3ee3709 --- /dev/null +++ b/apps/trip-cluster/categorize.py @@ -0,0 +1,5 @@ +import sys +from app.cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/apps/trip-cluster/pyproject.toml b/apps/trip-cluster/pyproject.toml new file mode 100644 index 0000000..f682724 --- /dev/null +++ b/apps/trip-cluster/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "trip-cluster" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = ["photoflow", "flask==3.1.0", "requests==2.32.3", "Pillow==11.0.0"] + +[tool.setuptools.packages.find] +where = ["."] +include = ["app*"] diff --git a/apps/trip-cluster/tests/conftest.py b/apps/trip-cluster/tests/conftest.py new file mode 100644 index 0000000..ac60ed2 --- /dev/null +++ b/apps/trip-cluster/tests/conftest.py @@ -0,0 +1 @@ +# Editable installs put `app` and `photoflow` on sys.path; nothing extra needed. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f3e7c59 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.pytest.ini_options] +testpaths = ["shared/tests", "apps/trip-cluster/tests"] +addopts = "-q" diff --git a/shared/photoflow/__init__.py b/shared/photoflow/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shared/photoflow/core/__init__.py b/shared/photoflow/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shared/photoflow/immich/__init__.py b/shared/photoflow/immich/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shared/photoflow/ui/__init__.py b/shared/photoflow/ui/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/shared/pyproject.toml b/shared/pyproject.toml new file mode 100644 index 0000000..7ed6113 --- /dev/null +++ b/shared/pyproject.toml @@ -0,0 +1,16 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "photoflow" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = ["requests==2.32.3", "flask==3.1.0", "Pillow==11.0.0"] + +[tool.setuptools.packages.find] +where = ["."] +include = ["photoflow*"] + +[tool.setuptools.package-data] +"photoflow.ui" = ["templates/*.html", "static/*.js"] diff --git a/shared/tests/test_imports.py b/shared/tests/test_imports.py new file mode 100644 index 0000000..7baa49e --- /dev/null +++ b/shared/tests/test_imports.py @@ -0,0 +1,4 @@ +def test_shared_packages_import(): + import photoflow.immich + import photoflow.core + import photoflow.ui -- 2.54.0 From 862480916acca50f5a777c64caa23205274c2a0d Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:11:49 +0200 Subject: [PATCH 02/19] =?UTF-8?q?feat(immich):=20ImmichClient=20read=20?= =?UTF-8?q?=E2=80=94=20search=5Fassets,=20list=5Ftags,=20resolve=5Ftag=5Fi?= =?UTF-8?q?d,=20download=5Fthumbnail?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- shared/photoflow/immich/__init__.py | 3 ++ shared/photoflow/immich/client.py | 74 +++++++++++++++++++++++++++++ shared/tests/test_immich_client.py | 50 +++++++++++++++++++ 3 files changed, 127 insertions(+) create mode 100644 shared/photoflow/immich/client.py create mode 100644 shared/tests/test_immich_client.py diff --git a/shared/photoflow/immich/__init__.py b/shared/photoflow/immich/__init__.py index e69de29..1f58377 100644 --- a/shared/photoflow/immich/__init__.py +++ b/shared/photoflow/immich/__init__.py @@ -0,0 +1,3 @@ +from photoflow.immich.client import ImmichClient + +__all__ = ["ImmichClient"] diff --git a/shared/photoflow/immich/client.py b/shared/photoflow/immich/client.py new file mode 100644 index 0000000..e8413ee --- /dev/null +++ b/shared/photoflow/immich/client.py @@ -0,0 +1,74 @@ +import requests + + +def _normalize(item: dict) -> dict: + exif = item.get("exifInfo") or {} + lat = exif.get("latitude") + lon = exif.get("longitude") + return { + "id": item["id"], + "original_filename": item.get("originalFileName", ""), + "taken_at": item.get("localDateTime", ""), + "gps_lat": float(lat) if lat is not None else None, + "gps_lon": float(lon) if lon is not None else None, + "place_city": exif.get("city"), + "place_country": exif.get("country"), + "type": item.get("type", ""), + "tags": [t.get("name", "") for t in (item.get("tags") or [])], + "rating": int(exif.get("rating") or 0), + "updated_at": item.get("updatedAt", ""), + } + + +class ImmichClient: + def __init__(self, base_url: str, api_key: str, timeout: int = 30): + self.base_url = base_url.rstrip("/") + self.timeout = timeout + self.session = requests.Session() + self.session.headers.update({"x-api-key": api_key, "Accept": "application/json"}) + + def _url(self, path: str) -> str: + return f"{self.base_url}{path}" + + def list_tags(self) -> list[dict]: + r = self.session.get(self._url("/api/tags"), timeout=self.timeout) + r.raise_for_status() + return r.json() + + def resolve_tag_id(self, name: str): + for tag in self.list_tags(): + if tag.get("value") == name or tag.get("name") == name: + return tag["id"] + return None + + def search_assets(self, *, taken_after=None, taken_before=None, + tag_ids=None, updated_after=None) -> list[dict]: + body = {"withExif": True} + if taken_after: + body["takenAfter"] = taken_after + if taken_before: + body["takenBefore"] = taken_before + if tag_ids: + body["tagIds"] = tag_ids + if updated_after: + body["updatedAfter"] = updated_after + out = [] + page = 1 + while True: + payload = dict(body, size=1000, page=page) + r = self.session.post(self._url("/api/search/metadata"), + json=payload, timeout=self.timeout) + r.raise_for_status() + block = r.json().get("assets", {}) + out.extend(_normalize(it) for it in block.get("items", [])) + nxt = block.get("nextPage") + if not nxt: + break + page = int(nxt) + return out + + def download_thumbnail(self, asset_id: str) -> bytes: + r = self.session.get(self._url(f"/api/assets/{asset_id}/thumbnail?size=preview"), + timeout=self.timeout) + r.raise_for_status() + return r.content diff --git a/shared/tests/test_immich_client.py b/shared/tests/test_immich_client.py new file mode 100644 index 0000000..547896e --- /dev/null +++ b/shared/tests/test_immich_client.py @@ -0,0 +1,50 @@ +from werkzeug.wrappers import Response +from photoflow.immich import ImmichClient + + +def test_resolve_tag_id_matches_name_or_value(httpserver): + httpserver.expect_request("/api/tags").respond_with_json([ + {"id": "t1", "name": "Italy 2019", "value": "Italy 2019"}, + {"id": "t2", "name": "processed", "value": "_pipeline/processed"}, + ]) + c = ImmichClient(httpserver.url_for(""), "k") + assert c.resolve_tag_id("Italy 2019") == "t1" + assert c.resolve_tag_id("_pipeline/processed") == "t2" + assert c.resolve_tag_id("nope") is None + + +def test_search_assets_normalizes_and_paginates(httpserver): + def handler(request): + page = request.json.get("page", 1) + assert request.json.get("withExif") is True + if page == 1: + return Response( + '{"assets": {"items": [{"id": "a", "originalFileName": "a.jpg",' + ' "localDateTime": "2019-06-01T10:00:00.000Z", "type": "IMAGE",' + ' "updatedAt": "2026-01-01T00:00:00Z",' + ' "exifInfo": {"latitude": 45.4, "longitude": 12.3, "city": "Venezia",' + ' "country": "Italy", "rating": 4},' + ' "tags": [{"name": "Italy 2019"}]}], "nextPage": 2}}', + content_type="application/json") + return Response( + '{"assets": {"items": [{"id": "b", "originalFileName": "b.jpg",' + ' "localDateTime": "2019-06-02T11:00:00.000Z", "type": "IMAGE",' + ' "updatedAt": "2026-01-02T00:00:00Z", "exifInfo": {}, "tags": []}],' + ' "nextPage": null}}', content_type="application/json") + + httpserver.expect_request("/api/search/metadata", method="POST").respond_with_handler(handler) + c = ImmichClient(httpserver.url_for(""), "k") + assets = c.search_assets(taken_after="2019-01-01", taken_before="2020-01-01") + assert [a["id"] for a in assets] == ["a", "b"] + a = assets[0] + assert a["gps_lat"] == 45.4 and a["place_city"] == "Venezia" + assert a["tags"] == ["Italy 2019"] and a["rating"] == 4 + assert a["taken_at"] == "2019-06-01T10:00:00.000Z" + assert assets[1]["gps_lat"] is None and assets[1]["tags"] == [] + + +def test_download_thumbnail(httpserver): + httpserver.expect_request("/api/assets/a/thumbnail").respond_with_data( + b"\xff\xd8\xffjpegbytes", content_type="image/jpeg") + c = ImmichClient(httpserver.url_for(""), "k") + assert c.download_thumbnail("a").startswith(b"\xff\xd8\xff") -- 2.54.0 From 4db684fbdca0a44ec0b41d72699bf1df98c8b992 Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:11:49 +0200 Subject: [PATCH 03/19] feat(core): domain models + Store (schema, connection, meta) --- shared/photoflow/core/__init__.py | 3 + shared/photoflow/core/models.py | 66 ++++++++++++++++++ shared/photoflow/core/store.py | 108 ++++++++++++++++++++++++++++++ shared/tests/test_store_assets.py | 24 +++++++ 4 files changed, 201 insertions(+) create mode 100644 shared/photoflow/core/models.py create mode 100644 shared/photoflow/core/store.py create mode 100644 shared/tests/test_store_assets.py diff --git a/shared/photoflow/core/__init__.py b/shared/photoflow/core/__init__.py index e69de29..187dad4 100644 --- a/shared/photoflow/core/__init__.py +++ b/shared/photoflow/core/__init__.py @@ -0,0 +1,3 @@ +from photoflow.core.store import Store, SCHEMA_VERSION + +__all__ = ["Store", "SCHEMA_VERSION"] diff --git a/shared/photoflow/core/models.py b/shared/photoflow/core/models.py new file mode 100644 index 0000000..8e9fd1e --- /dev/null +++ b/shared/photoflow/core/models.py @@ -0,0 +1,66 @@ +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class Asset: + immich_id: str + taken_at: str = "" + gps_lat: Optional[float] = None + gps_lon: Optional[float] = None + place_city: Optional[str] = None + place_country: Optional[str] = None + type: str = "" + has_gps: bool = False + thumb_path: Optional[str] = None + processed: bool = False + ingested_at: str = "" + updated_at: str = "" + + +@dataclass +class Tag: + name: str + immich_tag_id: Optional[str] = None + count: int = 0 + + +@dataclass +class AssetTag: + immich_id: str + tag_name: str + + +@dataclass +class Cluster: + id: Optional[int] = None + start_at: str = "" + end_at: str = "" + count: int = 0 + suggested_name: str = "" + confidence: float = 0.0 + kind_guess: str = "trip" # trip | everyday + status: str = "pending" # pending|approved|non_trip|merged|split|skipped + decided_name: Optional[str] = None + reviewed_at: Optional[str] = None + notes: Optional[str] = None + + +@dataclass +class ClusterMember: + cluster_id: int + immich_id: str + member_confidence: float = 1.0 + is_outlier: bool = False + included: bool = True + flagged_coverage: bool = False + + +@dataclass +class WritebackLog: + id: Optional[int] + immich_id: str + action: str # trip | non-trip | processed + tag: Optional[str] + result: str # ok | error: + applied_at: str diff --git a/shared/photoflow/core/store.py b/shared/photoflow/core/store.py new file mode 100644 index 0000000..a7877d3 --- /dev/null +++ b/shared/photoflow/core/store.py @@ -0,0 +1,108 @@ +import sqlite3 + +SCHEMA_VERSION = 1 + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT); + +CREATE TABLE IF NOT EXISTS assets ( + immich_id TEXT PRIMARY KEY, + taken_at TEXT, + gps_lat REAL, gps_lon REAL, + place_city TEXT, place_country TEXT, + type TEXT, + has_gps INTEGER NOT NULL DEFAULT 0, + thumb_path TEXT, + processed INTEGER NOT NULL DEFAULT 0, + ingested_at TEXT, + updated_at TEXT +); + +CREATE TABLE IF NOT EXISTS tags ( + name TEXT PRIMARY KEY, + immich_tag_id TEXT, + count INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE IF NOT EXISTS asset_tags ( + immich_id TEXT NOT NULL, + tag_name TEXT NOT NULL, + PRIMARY KEY (immich_id, tag_name) +); + +CREATE TABLE IF NOT EXISTS clusters ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + start_at TEXT, end_at TEXT, + count INTEGER NOT NULL DEFAULT 0, + suggested_name TEXT, + confidence REAL NOT NULL DEFAULT 0, + kind_guess TEXT NOT NULL DEFAULT 'trip', + status TEXT NOT NULL DEFAULT 'pending', + decided_name TEXT, + reviewed_at TEXT, + notes TEXT +); + +CREATE TABLE IF NOT EXISTS cluster_members ( + cluster_id INTEGER NOT NULL, + immich_id TEXT NOT NULL, + member_confidence REAL NOT NULL DEFAULT 1.0, + is_outlier INTEGER NOT NULL DEFAULT 0, + included INTEGER NOT NULL DEFAULT 1, + flagged_coverage INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (cluster_id, immich_id) +); + +CREATE TABLE IF NOT EXISTS writeback_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + immich_id TEXT NOT NULL, + action TEXT NOT NULL, + tag TEXT, + result TEXT NOT NULL, + applied_at TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_assets_taken_at ON assets(taken_at); +CREATE INDEX IF NOT EXISTS idx_members_cluster ON cluster_members(cluster_id); +CREATE INDEX IF NOT EXISTS idx_members_asset ON cluster_members(immich_id); +""" + + +class Store: + def __init__(self, db_path: str): + self.db_path = db_path + self._conn = None + + def connect(self) -> "Store": + self._conn = sqlite3.connect(self.db_path) + self._conn.row_factory = sqlite3.Row + self._conn.execute("PRAGMA foreign_keys=ON") + self.migrate() + return self + + @property + def conn(self) -> sqlite3.Connection: + if self._conn is None: + raise RuntimeError("Store not connected; call connect()") + return self._conn + + def migrate(self) -> None: + self.conn.executescript(SCHEMA) + if self.get_meta("schema_version") is None: + self.set_meta("schema_version", str(SCHEMA_VERSION)) + self.conn.commit() + + def set_meta(self, key: str, value: str) -> None: + self.conn.execute( + "INSERT INTO meta(key, value) VALUES(?, ?) " + "ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, value)) + self.conn.commit() + + def get_meta(self, key: str, default=None): + row = self.conn.execute("SELECT value FROM meta WHERE key=?", (key,)).fetchone() + return row["value"] if row else default + + def close(self) -> None: + if self._conn is not None: + self._conn.close() + self._conn = None diff --git a/shared/tests/test_store_assets.py b/shared/tests/test_store_assets.py new file mode 100644 index 0000000..f60a65b --- /dev/null +++ b/shared/tests/test_store_assets.py @@ -0,0 +1,24 @@ +from photoflow.core import Store + + +def test_connect_creates_schema_and_version(tmp_path): + db = str(tmp_path / "t.db") + s = Store(db).connect() + assert s.get_meta("schema_version") == "1" + # tables exist + names = {r["name"] for r in s.conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'")} + assert {"assets", "tags", "asset_tags", "clusters", + "cluster_members", "writeback_log", "meta"} <= names + s.close() + + +def test_meta_roundtrip_and_default(tmp_path): + s = Store(str(tmp_path / "t.db")).connect() + assert s.get_meta("missing") is None + assert s.get_meta("missing", "x") == "x" + s.set_meta("last_ingest_at", "2026-06-27T00:00:00Z") + assert s.get_meta("last_ingest_at") == "2026-06-27T00:00:00Z" + s.set_meta("last_ingest_at", "newer") # upsert + assert s.get_meta("last_ingest_at") == "newer" + s.close() -- 2.54.0 From c36d93e3c7ece9d8f33dab8dcc350f90872314ab Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:11:49 +0200 Subject: [PATCH 04/19] feat(ui): base.html, macros (badges, lightbox), shared.js, register_shared_ui --- shared/photoflow/ui/__init__.py | 16 +++++++ shared/photoflow/ui/static/shared.js | 57 +++++++++++++++++++++++ shared/photoflow/ui/templates/base.html | 24 ++++++++++ shared/photoflow/ui/templates/macros.html | 30 ++++++++++++ shared/tests/test_ui.py | 24 ++++++++++ 5 files changed, 151 insertions(+) create mode 100644 shared/photoflow/ui/static/shared.js create mode 100644 shared/photoflow/ui/templates/base.html create mode 100644 shared/photoflow/ui/templates/macros.html create mode 100644 shared/tests/test_ui.py diff --git a/shared/photoflow/ui/__init__.py b/shared/photoflow/ui/__init__.py index e69de29..f0f2bae 100644 --- a/shared/photoflow/ui/__init__.py +++ b/shared/photoflow/ui/__init__.py @@ -0,0 +1,16 @@ +import os +from flask import Blueprint, Flask +from jinja2 import ChoiceLoader, FileSystemLoader + +TEMPLATE_DIR = os.path.join(os.path.dirname(__file__), "templates") +STATIC_DIR = os.path.join(os.path.dirname(__file__), "static") + + +def register_shared_ui(app: Flask) -> None: + app.jinja_loader = ChoiceLoader([app.jinja_loader, FileSystemLoader(TEMPLATE_DIR)]) + bp = Blueprint("shared_ui", __name__, static_folder=STATIC_DIR, + static_url_path="/shared-static") + app.register_blueprint(bp) + + +__all__ = ["TEMPLATE_DIR", "STATIC_DIR", "register_shared_ui"] diff --git a/shared/photoflow/ui/static/shared.js b/shared/photoflow/ui/static/shared.js new file mode 100644 index 0000000..a5d77a1 --- /dev/null +++ b/shared/photoflow/ui/static/shared.js @@ -0,0 +1,57 @@ +// Generic grid + lightbox behavior shared across photoflow apps. +// Apps spread this into their own Alpine component: { ...photoGrid(), ...appLogic } +function photoGrid() { + return { + focused: null, + lightboxOpen: false, + + cards() { + return [...document.querySelectorAll('.photo-card')] + .filter(c => c.style.display !== 'none'); + }, + + select(el) { + if (this.focused) this.focused.classList.remove('ring-4', 'ring-white', 'z-10'); + this.focused = el; + if (el) { + el.classList.add('ring-4', 'ring-white', 'z-10'); + el.scrollIntoView({ behavior: 'smooth', block: 'nearest' }); + if (this.lightboxOpen) this.updateLightbox(); + } + }, + + selectFirst() { + const cards = this.cards(); + if (cards.length) this.select(cards[0]); + }, + + navigate(dir) { + const cards = this.cards(); + if (!cards.length) return; + const idx = this.focused ? cards.indexOf(this.focused) : -1; + const next = cards[Math.max(0, Math.min(cards.length - 1, idx + dir))]; + if (next) this.select(next); + }, + + openLightbox(el) { + this.select(el); + this.lightboxOpen = true; + document.getElementById('lb').style.display = ''; + this.updateLightbox(); + }, + + closeLightbox() { + this.lightboxOpen = false; + const lb = document.getElementById('lb'); + if (lb) lb.style.display = 'none'; + }, + + updateLightbox() { + const el = this.focused; + if (!el) return; + const img = el.querySelector('img'); + const lbImg = document.getElementById('lb-img'); + if (img && lbImg) lbImg.src = img.src; + }, + }; +} diff --git a/shared/photoflow/ui/templates/base.html b/shared/photoflow/ui/templates/base.html new file mode 100644 index 0000000..249c29f --- /dev/null +++ b/shared/photoflow/ui/templates/base.html @@ -0,0 +1,24 @@ + + + + + + {% block title %}photoflow{% endblock %} + + + + + + +
+
+ {% block navbar_title %}photoflow{% endblock %} +
+
+
+ {% block content %}{% endblock %} +
+ + {% block extra_scripts %}{% endblock %} + + diff --git a/shared/photoflow/ui/templates/macros.html b/shared/photoflow/ui/templates/macros.html new file mode 100644 index 0000000..76430b0 --- /dev/null +++ b/shared/photoflow/ui/templates/macros.html @@ -0,0 +1,30 @@ +{% macro status_badge(status) %} + {% set cls = {'pending':'badge-ghost','approved':'badge-success','non_trip':'badge-neutral', + 'skipped':'badge-warning','merged':'badge-info','split':'badge-info'} %} + {{ status }} +{% endmacro %} + +{% macro confidence_badge(confidence) %} + {% if confidence < 0.4 %} + low + {% elif confidence < 0.75 %} + med + {% else %} + high + {% endif %} +{% endmacro %} + +{% macro lightbox() %} + +{% endmacro %} diff --git a/shared/tests/test_ui.py b/shared/tests/test_ui.py new file mode 100644 index 0000000..2bdbf89 --- /dev/null +++ b/shared/tests/test_ui.py @@ -0,0 +1,24 @@ +import os +from flask import Flask, render_template +from photoflow.ui import TEMPLATE_DIR, STATIC_DIR, register_shared_ui + + +def test_dirs_exist(): + assert os.path.isfile(os.path.join(TEMPLATE_DIR, "base.html")) + assert os.path.isfile(os.path.join(STATIC_DIR, "shared.js")) + + +def test_register_serves_shared_static_and_template(): + app = Flask(__name__) + register_shared_ui(app) + + @app.route("/page") + def page(): + return render_template("base.html") + + client = app.test_client() + r = client.get("/page") + assert r.status_code == 200 + assert b"/shared-static/shared.js" in r.data + js = client.get("/shared-static/shared.js") + assert js.status_code == 200 and b"photoGrid" in js.data -- 2.54.0 From da5b1de2a20787a9f43e67045dca1bef170e6ecd Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:16:07 +0200 Subject: [PATCH 05/19] feat(immich): write-back (upsert_tag, tag_assets) + _pipeline tag conventions --- shared/photoflow/immich/__init__.py | 3 ++- shared/photoflow/immich/client.py | 19 +++++++++++++++++++ shared/photoflow/immich/pipeline.py | 11 +++++++++++ shared/tests/test_immich_client.py | 13 +++++++++++++ shared/tests/test_pipeline.py | 15 +++++++++++++++ 5 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 shared/photoflow/immich/pipeline.py create mode 100644 shared/tests/test_pipeline.py diff --git a/shared/photoflow/immich/__init__.py b/shared/photoflow/immich/__init__.py index 1f58377..c6028b4 100644 --- a/shared/photoflow/immich/__init__.py +++ b/shared/photoflow/immich/__init__.py @@ -1,3 +1,4 @@ from photoflow.immich.client import ImmichClient +from photoflow.immich import pipeline -__all__ = ["ImmichClient"] +__all__ = ["ImmichClient", "pipeline"] diff --git a/shared/photoflow/immich/client.py b/shared/photoflow/immich/client.py index e8413ee..0ec40d1 100644 --- a/shared/photoflow/immich/client.py +++ b/shared/photoflow/immich/client.py @@ -72,3 +72,22 @@ class ImmichClient: timeout=self.timeout) r.raise_for_status() return r.content + + def upsert_tag(self, name: str) -> str: + r = self.session.put(self._url("/api/tags"), + json={"tags": [name]}, timeout=self.timeout) + r.raise_for_status() + for tag in r.json(): + if tag.get("value") == name or tag.get("name") == name: + return tag["id"] + resolved = self.resolve_tag_id(name) + if resolved is None: + raise RuntimeError(f"upsert_tag: could not resolve id for {name!r}") + return resolved + + def tag_assets(self, tag_id: str, asset_ids: list[str]) -> None: + if not asset_ids: + return + r = self.session.put(self._url(f"/api/tags/{tag_id}/assets"), + json={"ids": asset_ids}, timeout=self.timeout) + r.raise_for_status() diff --git a/shared/photoflow/immich/pipeline.py b/shared/photoflow/immich/pipeline.py new file mode 100644 index 0000000..b2c4c42 --- /dev/null +++ b/shared/photoflow/immich/pipeline.py @@ -0,0 +1,11 @@ +ROOT = "_pipeline" +PROCESSED = f"{ROOT}/processed" +NON_TRIP = f"{ROOT}/non-trip" + + +def ai_rating(n: int) -> str: + return f"{ROOT}/ai-rating/{n}" + + +def is_pipeline_tag(name: str) -> bool: + return name == ROOT or name.startswith(ROOT + "/") diff --git a/shared/tests/test_immich_client.py b/shared/tests/test_immich_client.py index 547896e..fe8e5b0 100644 --- a/shared/tests/test_immich_client.py +++ b/shared/tests/test_immich_client.py @@ -48,3 +48,16 @@ def test_download_thumbnail(httpserver): b"\xff\xd8\xffjpegbytes", content_type="image/jpeg") c = ImmichClient(httpserver.url_for(""), "k") assert c.download_thumbnail("a").startswith(b"\xff\xd8\xff") + + +def test_upsert_tag_returns_id_by_value(httpserver): + httpserver.expect_request("/api/tags", method="PUT").respond_with_json( + [{"id": "p1", "name": "non-trip", "value": "_pipeline/non-trip"}]) + c = ImmichClient(httpserver.url_for(""), "k") + assert c.upsert_tag("_pipeline/non-trip") == "p1" + + +def test_tag_assets_posts_ids(httpserver): + httpserver.expect_request("/api/tags/p1/assets", method="PUT").respond_with_json({"ok": True}) + c = ImmichClient(httpserver.url_for(""), "k") + c.tag_assets("p1", ["a", "b"]) # should not raise diff --git a/shared/tests/test_pipeline.py b/shared/tests/test_pipeline.py new file mode 100644 index 0000000..35867f1 --- /dev/null +++ b/shared/tests/test_pipeline.py @@ -0,0 +1,15 @@ +from photoflow.immich import pipeline + + +def test_constants_and_helpers(): + assert pipeline.ROOT == "_pipeline" + assert pipeline.PROCESSED == "_pipeline/processed" + assert pipeline.NON_TRIP == "_pipeline/non-trip" + assert pipeline.ai_rating(4) == "_pipeline/ai-rating/4" + + +def test_is_pipeline_tag(): + assert pipeline.is_pipeline_tag("_pipeline/processed") is True + assert pipeline.is_pipeline_tag("_pipeline") is True + assert pipeline.is_pipeline_tag("Italy 2019") is False + assert pipeline.is_pipeline_tag("Kiev") is False -- 2.54.0 From 351a001903c2217cd09a1d87148a4830251b08b0 Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:16:07 +0200 Subject: [PATCH 06/19] feat(core): asset & tag data-access --- shared/photoflow/core/store.py | 73 +++++++++++++++++++++++++++++++ shared/tests/test_store_assets.py | 57 ++++++++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/shared/photoflow/core/store.py b/shared/photoflow/core/store.py index a7877d3..3cf889c 100644 --- a/shared/photoflow/core/store.py +++ b/shared/photoflow/core/store.py @@ -1,5 +1,7 @@ import sqlite3 +from photoflow.core.models import Asset, Tag + SCHEMA_VERSION = 1 SCHEMA = """ @@ -106,3 +108,74 @@ class Store: if self._conn is not None: self._conn.close() self._conn = None + + def upsert_asset(self, a: Asset) -> None: + has_gps = 1 if (a.gps_lat is not None and a.gps_lon is not None) else 0 + self.conn.execute( + """INSERT INTO assets + (immich_id, taken_at, gps_lat, gps_lon, place_city, place_country, + type, has_gps, thumb_path, processed, ingested_at, updated_at) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?) + ON CONFLICT(immich_id) DO UPDATE SET + taken_at=excluded.taken_at, gps_lat=excluded.gps_lat, + gps_lon=excluded.gps_lon, place_city=excluded.place_city, + place_country=excluded.place_country, type=excluded.type, + has_gps=excluded.has_gps, thumb_path=excluded.thumb_path, + ingested_at=excluded.ingested_at, updated_at=excluded.updated_at""", + (a.immich_id, a.taken_at, a.gps_lat, a.gps_lon, a.place_city, + a.place_country, a.type, has_gps, a.thumb_path, + 1 if a.processed else 0, a.ingested_at, a.updated_at)) + self.conn.commit() + + def _asset_from_row(self, r) -> Asset: + return Asset( + immich_id=r["immich_id"], taken_at=r["taken_at"], + gps_lat=r["gps_lat"], gps_lon=r["gps_lon"], + place_city=r["place_city"], place_country=r["place_country"], + type=r["type"], has_gps=bool(r["has_gps"]), thumb_path=r["thumb_path"], + processed=bool(r["processed"]), ingested_at=r["ingested_at"], + updated_at=r["updated_at"]) + + def get_asset(self, immich_id: str): + r = self.conn.execute("SELECT * FROM assets WHERE immich_id=?", (immich_id,)).fetchone() + return self._asset_from_row(r) if r else None + + def all_assets(self, include_processed: bool = True) -> list: + sql = "SELECT * FROM assets" + if not include_processed: + sql += " WHERE processed=0" + sql += " ORDER BY taken_at" + return [self._asset_from_row(r) for r in self.conn.execute(sql)] + + def assets_in_range(self, start_at: str, end_at: str) -> list: + return [self._asset_from_row(r) for r in self.conn.execute( + "SELECT * FROM assets WHERE taken_at>=? AND taken_at<=? ORDER BY taken_at", + (start_at, end_at))] + + def mark_processed(self, immich_id: str) -> None: + self.conn.execute("UPDATE assets SET processed=1 WHERE immich_id=?", (immich_id,)) + self.conn.commit() + + def set_asset_tags(self, immich_id: str, tag_names: list) -> None: + self.conn.execute("DELETE FROM asset_tags WHERE immich_id=?", (immich_id,)) + self.conn.executemany( + "INSERT OR IGNORE INTO asset_tags(immich_id, tag_name) VALUES(?, ?)", + [(immich_id, n) for n in tag_names]) + self.conn.commit() + + def asset_tags(self, immich_id: str) -> list: + return [r["tag_name"] for r in self.conn.execute( + "SELECT tag_name FROM asset_tags WHERE immich_id=? ORDER BY tag_name", + (immich_id,))] + + def upsert_tag(self, name: str, immich_tag_id=None, count: int = 0) -> None: + self.conn.execute( + """INSERT INTO tags(name, immich_tag_id, count) VALUES(?,?,?) + ON CONFLICT(name) DO UPDATE SET + immich_tag_id=excluded.immich_tag_id, count=excluded.count""", + (name, immich_tag_id, count)) + self.conn.commit() + + def all_tags(self) -> list: + return [Tag(name=r["name"], immich_tag_id=r["immich_tag_id"], count=r["count"]) + for r in self.conn.execute("SELECT * FROM tags ORDER BY name")] diff --git a/shared/tests/test_store_assets.py b/shared/tests/test_store_assets.py index f60a65b..09bedad 100644 --- a/shared/tests/test_store_assets.py +++ b/shared/tests/test_store_assets.py @@ -22,3 +22,60 @@ def test_meta_roundtrip_and_default(tmp_path): s.set_meta("last_ingest_at", "newer") # upsert assert s.get_meta("last_ingest_at") == "newer" s.close() + + +from photoflow.core.models import Asset + + +def _store(tmp_path): + return Store(str(tmp_path / "t.db")).connect() + + +def test_upsert_asset_roundtrip_and_has_gps(tmp_path): + s = _store(tmp_path) + s.upsert_asset(Asset(immich_id="a", taken_at="2019-06-01T10:00:00", + gps_lat=45.4, gps_lon=12.3, place_city="Venezia", type="IMAGE")) + got = s.get_asset("a") + assert got.place_city == "Venezia" and got.has_gps is True + s.upsert_asset(Asset(immich_id="b", taken_at="2019-06-02T10:00:00")) + assert s.get_asset("b").has_gps is False + s.close() + + +def test_upsert_preserves_processed(tmp_path): + s = _store(tmp_path) + s.upsert_asset(Asset(immich_id="a", taken_at="t")) + s.mark_processed("a") + s.upsert_asset(Asset(immich_id="a", taken_at="t2")) # re-ingest + got = s.get_asset("a") + assert got.processed is True and got.taken_at == "t2" + s.close() + + +def test_all_assets_ordered_and_range(tmp_path): + s = _store(tmp_path) + for i, t in [("c", "2019-06-03"), ("a", "2019-06-01"), ("b", "2019-06-02")]: + s.upsert_asset(Asset(immich_id=i, taken_at=t)) + assert [a.immich_id for a in s.all_assets()] == ["a", "b", "c"] + rng = s.assets_in_range("2019-06-02", "2019-06-03") + assert [a.immich_id for a in rng] == ["b", "c"] + s.close() + + +def test_asset_tags_replace(tmp_path): + s = _store(tmp_path) + s.upsert_asset(Asset(immich_id="a", taken_at="t")) + s.set_asset_tags("a", ["Italy 2019", "Kiev"]) + assert sorted(s.asset_tags("a")) == ["Italy 2019", "Kiev"] + s.set_asset_tags("a", ["Italy 2019"]) # replace + assert s.asset_tags("a") == ["Italy 2019"] + s.close() + + +def test_tags_inventory(tmp_path): + s = _store(tmp_path) + s.upsert_tag("Italy 2019", immich_tag_id="t1", count=42) + s.upsert_tag("Italy 2019", immich_tag_id="t1", count=43) # upsert + names = {t.name: t for t in s.all_tags()} + assert names["Italy 2019"].count == 43 and names["Italy 2019"].immich_tag_id == "t1" + s.close() -- 2.54.0 From 76be1fb5af2dec5e0e7b3e9aa8103c32cb7a1f73 Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:16:07 +0200 Subject: [PATCH 07/19] feat(trip-cluster): density-adaptive timestamp clustering + tag seeds + anchors --- apps/trip-cluster/app/clustering.py | 192 +++++++++++++++++++++ apps/trip-cluster/tests/test_clustering.py | 62 +++++++ 2 files changed, 254 insertions(+) create mode 100644 apps/trip-cluster/app/clustering.py create mode 100644 apps/trip-cluster/tests/test_clustering.py diff --git a/apps/trip-cluster/app/clustering.py b/apps/trip-cluster/app/clustering.py new file mode 100644 index 0000000..5f35b28 --- /dev/null +++ b/apps/trip-cluster/app/clustering.py @@ -0,0 +1,192 @@ +import datetime +from collections import Counter +from dataclasses import dataclass +from typing import Optional + +EVERYDAY_MAX_COUNT = 4 +SEED_CONFIDENCE = 0.95 + +# Public aliases for cross-module use are defined at the bottom of this file +# (epoch / median) so a later coverage.py can consume them without reaching +# for the underscore-prefixed names. The underscore names are kept too, since +# Task 11 (coverage.py) imports `_epoch` / `_median` directly. +__all__ = [ + "CandidateCluster", + "cluster_assets", + "_epoch", + "_median", + "epoch", + "median", +] + + +@dataclass +class CandidateCluster: + member_ids: list + start_at: str + end_at: str + suggested_name: str + confidence: float + kind_guess: str + seed_tag: Optional[str] = None + + +def _epoch(taken_at: str) -> float: + s = (taken_at or "").strip() + if not s: + return 0.0 + s = s.replace("Z", "") + if "." in s: + s = s.split(".", 1)[0] + try: + if "T" in s: + return datetime.datetime.fromisoformat(s).timestamp() + return datetime.datetime.fromisoformat(s + "T00:00:00").timestamp() + except ValueError: + return 0.0 + + +def _median(values: list) -> float: + if not values: + return 0.0 + xs = sorted(values) + n = len(xs) + mid = n // 2 + return xs[mid] if n % 2 else (xs[mid - 1] + xs[mid]) / 2 + + +def _span(members: list) -> tuple: + ts = [m.taken_at for m in members] + return (min(ts), max(ts)) if ts else ("", "") + + +def _name(members: list, start_at: str) -> str: + cities = Counter(m.place_city for m in members if m.place_city) + if cities: + return cities.most_common(1)[0][0] + countries = Counter(m.place_country for m in members if m.place_country) + if countries: + return countries.most_common(1)[0][0] + return f"Trip {start_at[:10]}" + + +def _tightness(members: list) -> float: + """Temporal-tightness score in [0, 1]: how regular the intra-cluster time + gaps are (low gap variance -> high score). + + A densely/regularly shot cluster (e.g. a steady stream of photos through a + day) is strong evidence of a coherent event even when GPS is absent. We + measure regularity via the coefficient of variation (stdev / mean) of the + consecutive gaps and reward a low value. This lets a GPS-poor but tightly + packed cluster clear the downstream 0.75 bulk-approve gate, which the + pure GPS+size score could never reach (it caps at 0.50 when gps_frac == 0). + """ + ts = sorted(_epoch(m.taken_at) for m in members) + gaps = [b - a for a, b in zip(ts, ts[1:])] + if not gaps: + return 0.0 # single member: no temporal signal + if len(gaps) < 2: + return 1.0 # one gap: trivially regular + mean = sum(gaps) / len(gaps) + if mean <= 0: + return 1.0 # all timestamps coincide: maximally tight + var = sum((g - mean) ** 2 for g in gaps) / len(gaps) + cv = (var ** 0.5) / mean + return max(0.0, 1.0 - cv) + + +def _confidence(members: list) -> float: + """Heuristic confidence in [0, 0.85] that a free cluster is a real event. + + Term rationale: + - 0.30 base: even a bare timestamp cluster is a weak positive signal, so + we never start from zero. + - 0.40 * gps_frac: geotagging is the strongest single signal that photos + belong to one outing, hence the largest weight. + - 0.20 * size_frac: more photos (saturating at 20) make a stray-photo + false positive less likely. + - 0.35 * tightness: regular/dense timing is independent evidence of a + coherent event; weighted so a fully GPS-poor cluster can still reach + the 0.85 cap (0.30 + 0.20 + 0.35) and clear the 0.75 approve gate. + The 0.85 cap reserves >0.90 confidence exclusively for tag-seeded clusters. + """ + count = len(members) + gps_frac = sum(1 for m in members if m.gps_lat is not None) / count if count else 0 + size_frac = min(count / 20, 1) + conf = 0.30 + 0.40 * gps_frac + 0.20 * size_frac + 0.35 * _tightness(members) + return round(min(conf, 0.85), 2) + + +def _free_cluster(members: list) -> CandidateCluster: + start, end = _span(members) + return CandidateCluster( + member_ids=[m.immich_id for m in members], + start_at=start, end_at=end, + suggested_name=_name(members, start), + confidence=_confidence(members), + kind_guess="everyday" if len(members) <= EVERYDAY_MAX_COUNT else "trip") + + +def _gap_cluster(assets: list, *, gap_factor, hard_split_days, min_floor_seconds) -> list: + ordered = sorted(assets, key=lambda a: a.taken_at) + if not ordered: + return [] + hard_cap = hard_split_days * 86400 + groups = [] + group = [ordered[0]] + group_gaps: list = [] + for prev, cur in zip(ordered, ordered[1:]): + gap = _epoch(cur.taken_at) - _epoch(prev.taken_at) + if gap > hard_cap: + split = True + elif len(group_gaps) < 2: # bootstrap: accept first 2 gaps + split = False + else: + threshold = max(gap_factor * _median(group_gaps), min_floor_seconds) + split = gap > threshold + if split: + groups.append(group) + group = [cur] + group_gaps = [] + else: + group.append(cur) + group_gaps.append(gap) + groups.append(group) + return [_free_cluster(g) for g in groups] + + +def cluster_assets(assets, tags_by_asset, seed_tags, *, gap_factor=6.0, + hard_split_days=14, min_floor_seconds=3600) -> list: + by_id = {a.immich_id: a for a in assets} + used = set() + clusters = [] + + # 1. Seed clusters from existing trip tags (authoritative; never gap-split). + for tag in sorted(seed_tags): + members = [by_id[aid] for aid in by_id + if aid not in used and tag in tags_by_asset.get(aid, [])] + if not members: + continue + members.sort(key=lambda a: a.taken_at) + used.update(m.immich_id for m in members) + start, end = _span(members) + clusters.append(CandidateCluster( + member_ids=[m.immich_id for m in members], start_at=start, end_at=end, + suggested_name=tag, confidence=SEED_CONFIDENCE, kind_guess="trip", + seed_tag=tag)) + + # 2. Gap-cluster the remaining (free) assets. + free = [a for a in assets if a.immich_id not in used] + clusters.extend(_gap_cluster(free, gap_factor=gap_factor, + hard_split_days=hard_split_days, + min_floor_seconds=min_floor_seconds)) + + clusters.sort(key=lambda c: c.start_at) + return clusters + + +# Public aliases (Review revision 2): expose the timestamp/median helpers for +# cross-module reuse (e.g. coverage.py) without forcing callers onto the +# underscore-prefixed names. The underscore names remain importable. +epoch = _epoch +median = _median diff --git a/apps/trip-cluster/tests/test_clustering.py b/apps/trip-cluster/tests/test_clustering.py new file mode 100644 index 0000000..55f392a --- /dev/null +++ b/apps/trip-cluster/tests/test_clustering.py @@ -0,0 +1,62 @@ +from photoflow.core.models import Asset +from app.clustering import cluster_assets + + +def _a(i, taken, gps=False, city=None): + return Asset(immich_id=i, taken_at=taken, + gps_lat=45.0 if gps else None, gps_lon=12.0 if gps else None, + place_city=city) + + +def test_seed_tag_forms_one_cluster_not_gap_split(): + # Two assets months apart but sharing a trip tag -> ONE seeded cluster. + assets = [_a("a", "2019-06-01T10:00:00"), _a("b", "2019-09-01T10:00:00")] + tags = {"a": ["Italy 2019"], "b": ["Italy 2019"]} + clusters = cluster_assets(assets, tags, {"Italy 2019"}) + assert len(clusters) == 1 + c = clusters[0] + assert c.seed_tag == "Italy 2019" and sorted(c.member_ids) == ["a", "b"] + assert c.confidence >= 0.9 and c.suggested_name == "Italy 2019" + + +def test_sparse_old_regime_splits_on_adaptive_threshold(): + # ~1 day intra-trip gaps; trips separated by 10 days (< 14d hard cap), + # so only the adaptive rule can split them. + a = [_a(f"a{i}", f"2008-06-0{i+1}T12:00:00") for i in range(5)] # Jun 1..5 + b = [_a(f"b{i}", f"2008-06-1{i+5}T12:00:00") for i in range(3)] # Jun 15..17 + assets = a + b + tags = {x.immich_id: [] for x in assets} + clusters = cluster_assets(assets, tags, set()) + assert len(clusters) == 2 + assert sorted(clusters[0].member_ids) == ["a0", "a1", "a2", "a3", "a4"] + + +def test_dense_recent_regime_splits_on_adaptive_threshold(): + # Hourly bursts within a day; 2-day gap between days. + day1 = [_a(f"d{i}", f"2024-03-10T{10+i:02d}:00:00") for i in range(4)] + day3 = [_a(f"e{i}", f"2024-03-12T{10+i:02d}:00:00") for i in range(4)] + assets = day1 + day3 + tags = {x.immich_id: [] for x in assets} + clusters = cluster_assets(assets, tags, set()) + assert len(clusters) == 2 + assert sorted(clusters[0].member_ids) == ["d0", "d1", "d2", "d3"] + + +def test_location_anchor_names_and_gps_confidence(): + assets = [_a("a", "2020-05-01T10:00:00", gps=True, city="Kiev"), + _a("b", "2020-05-01T12:00:00", gps=True, city="Kiev"), + _a("c", "2020-05-01T14:00:00", gps=True, city="Kiev"), + _a("d", "2020-05-01T16:00:00", gps=True, city="Kiev"), + _a("e", "2020-05-01T18:00:00", gps=True, city="Kiev")] + tags = {x.immich_id: [] for x in assets} + c = cluster_assets(assets, tags, set())[0] + assert c.suggested_name == "Kiev" + assert c.confidence > 0.6 # full GPS lifts confidence + assert c.kind_guess == "trip" + + +def test_small_scattered_cluster_marked_everyday(): + assets = [_a("a", "2015-01-01T10:00:00"), _a("b", "2015-01-01T11:00:00")] + tags = {"a": [], "b": []} + c = cluster_assets(assets, tags, set())[0] + assert c.kind_guess == "everyday" -- 2.54.0 From 9ec65124284b0b88018b8d8a12185eab3da80dce Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:19:06 +0200 Subject: [PATCH 08/19] =?UTF-8?q?feat(core):=20cluster=20data-access=20?= =?UTF-8?q?=E2=80=94=20attention=20sort,=20neighbors,=20split/merge,=20wri?= =?UTF-8?q?teback=20log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- shared/photoflow/core/store.py | 158 +++++++++++++++++++++++++++- shared/tests/test_store_clusters.py | 99 +++++++++++++++++ 2 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 shared/tests/test_store_clusters.py diff --git a/shared/photoflow/core/store.py b/shared/photoflow/core/store.py index 3cf889c..32c67ab 100644 --- a/shared/photoflow/core/store.py +++ b/shared/photoflow/core/store.py @@ -1,6 +1,6 @@ import sqlite3 -from photoflow.core.models import Asset, Tag +from photoflow.core.models import Asset, Cluster, ClusterMember, Tag SCHEMA_VERSION = 1 @@ -179,3 +179,159 @@ class Store: def all_tags(self) -> list: return [Tag(name=r["name"], immich_tag_id=r["immich_tag_id"], count=r["count"]) for r in self.conn.execute("SELECT * FROM tags ORDER BY name")] + + def insert_cluster(self, c: Cluster, members: list) -> int: + cur = self.conn.execute( + """INSERT INTO clusters + (start_at, end_at, count, suggested_name, confidence, kind_guess, + status, decided_name, reviewed_at, notes) + VALUES (?,?,?,?,?,?,?,?,?,?)""", + (c.start_at, c.end_at, c.count or len(members), c.suggested_name, + c.confidence, c.kind_guess, c.status, c.decided_name, c.reviewed_at, c.notes)) + cid = cur.lastrowid + self.conn.executemany( + """INSERT OR REPLACE INTO cluster_members + (cluster_id, immich_id, member_confidence, is_outlier, included, flagged_coverage) + VALUES (?,?,?,?,?,?)""", + [(cid, m.immich_id, m.member_confidence, 1 if m.is_outlier else 0, + 1 if m.included else 0, 1 if m.flagged_coverage else 0) for m in members]) + self.conn.commit() + return cid + + def clear_clusters(self) -> None: + self.conn.execute("DELETE FROM cluster_members") + self.conn.execute("DELETE FROM clusters") + self.conn.commit() + + def _cluster_from_row(self, r) -> Cluster: + return Cluster( + id=r["id"], start_at=r["start_at"], end_at=r["end_at"], count=r["count"], + suggested_name=r["suggested_name"], confidence=r["confidence"], + kind_guess=r["kind_guess"], status=r["status"], decided_name=r["decided_name"], + reviewed_at=r["reviewed_at"], notes=r["notes"]) + + def get_cluster(self, cluster_id: int): + r = self.conn.execute("SELECT * FROM clusters WHERE id=?", (cluster_id,)).fetchone() + return self._cluster_from_row(r) if r else None + + def all_clusters(self) -> list: + return [self._cluster_from_row(r) + for r in self.conn.execute("SELECT * FROM clusters ORDER BY start_at, id")] + + def clusters_by_attention(self) -> list: + return [self._cluster_from_row(r) for r in self.conn.execute( + """SELECT * FROM clusters + ORDER BY CASE status WHEN 'pending' THEN 0 ELSE 1 END, + confidence ASC, + (julianday(end_at) - julianday(start_at)) DESC, + id""")] + + def cluster_members(self, cluster_id: int) -> list: + rows = self.conn.execute( + """SELECT a.*, m.member_confidence AS m_conf, m.is_outlier AS m_out, + m.included AS m_inc, m.flagged_coverage AS m_cov + FROM cluster_members m JOIN assets a ON a.immich_id = m.immich_id + WHERE m.cluster_id=? ORDER BY a.taken_at, a.immich_id""", (cluster_id,)) + out = [] + for r in rows: + asset = self._asset_from_row(r) + member = ClusterMember( + cluster_id=cluster_id, immich_id=r["immich_id"], + member_confidence=r["m_conf"], is_outlier=bool(r["m_out"]), + included=bool(r["m_inc"]), flagged_coverage=bool(r["m_cov"])) + out.append((asset, member)) + return out + + def chronological_neighbors(self, cluster_id: int): + ids = [r["id"] for r in self.conn.execute( + "SELECT id FROM clusters ORDER BY start_at, id")] + if cluster_id not in ids: + return (None, None) + i = ids.index(cluster_id) + prev_id = ids[i - 1] if i > 0 else None + next_id = ids[i + 1] if i < len(ids) - 1 else None + return (prev_id, next_id) + + def update_cluster(self, cluster_id: int, *, status=None, decided_name=None, + suggested_name=None, notes=None, reviewed_at=None) -> None: + sets, vals = [], [] + for col, val in [("status", status), ("decided_name", decided_name), + ("suggested_name", suggested_name), ("notes", notes), + ("reviewed_at", reviewed_at)]: + if val is not None: + sets.append(f"{col}=?") + vals.append(val) + if not sets: + return + vals.append(cluster_id) + self.conn.execute(f"UPDATE clusters SET {', '.join(sets)} WHERE id=?", vals) + self.conn.commit() + + def set_member_inclusion(self, cluster_id: int, immich_id: str, included: bool) -> None: + self.conn.execute( + "UPDATE cluster_members SET included=? WHERE cluster_id=? AND immich_id=?", + (1 if included else 0, cluster_id, immich_id)) + self.conn.commit() + + def _recompute_span(self, members: list) -> tuple: + pairs = members + starts = [a.taken_at for a, _ in pairs] + return (min(starts), max(starts)) if starts else ("", "") + + def split_cluster(self, cluster_id: int, boundary_immich_id: str): + pairs = self.cluster_members(cluster_id) + ids = [a.immich_id for a, _ in pairs] + if boundary_immich_id not in ids: + raise ValueError(f"boundary {boundary_immich_id!r} not in cluster") + idx = ids.index(boundary_immich_id) + base = self.get_cluster(cluster_id) + left, right = pairs[:idx], pairs[idx:] + + def _new(part, suffix): + ms = [ClusterMember(cluster_id=0, immich_id=a.immich_id, + member_confidence=m.member_confidence, + is_outlier=m.is_outlier, included=m.included, + flagged_coverage=m.flagged_coverage) for a, m in part] + start, end = self._recompute_span(part) + return self.insert_cluster(Cluster( + start_at=start, end_at=end, count=len(ms), + suggested_name=f"{base.suggested_name} ({suffix})", + confidence=base.confidence, kind_guess=base.kind_guess, + status="pending"), ms) + + id1, id2 = _new(left, 1), _new(right, 2) + self.update_cluster(cluster_id, status="split") + return (id1, id2) + + def merge_clusters(self, cluster_id_a: int, cluster_id_b: int): + pairs = self.cluster_members(cluster_id_a) + self.cluster_members(cluster_id_b) + pairs.sort(key=lambda p: (p[0].taken_at, p[0].immich_id)) + base = self.get_cluster(cluster_id_a) + ms = [ClusterMember(cluster_id=0, immich_id=a.immich_id, + member_confidence=m.member_confidence, is_outlier=m.is_outlier, + included=m.included, flagged_coverage=m.flagged_coverage) + for a, m in pairs] + start, end = self._recompute_span(pairs) + new = self.insert_cluster(Cluster( + start_at=start, end_at=end, count=len(ms), + suggested_name=base.suggested_name, confidence=base.confidence, + kind_guess=base.kind_guess, status="pending"), ms) + self.update_cluster(cluster_id_a, status="merged") + self.update_cluster(cluster_id_b, status="merged") + return new + + def log_writeback(self, immich_id: str, action: str, tag, result: str) -> None: + import datetime + self.conn.execute( + "INSERT INTO writeback_log(immich_id, action, tag, result, applied_at) " + "VALUES (?,?,?,?,?)", + (immich_id, action, tag, result, + datetime.datetime.now(datetime.timezone.utc).isoformat())) + self.conn.commit() + + def already_applied(self, immich_id: str, action: str, tag) -> bool: + row = self.conn.execute( + "SELECT 1 FROM writeback_log WHERE immich_id=? AND action=? " + "AND IFNULL(tag,'')=IFNULL(?, '') AND result='ok' LIMIT 1", + (immich_id, action, tag)).fetchone() + return row is not None diff --git a/shared/tests/test_store_clusters.py b/shared/tests/test_store_clusters.py new file mode 100644 index 0000000..d8bddd8 --- /dev/null +++ b/shared/tests/test_store_clusters.py @@ -0,0 +1,99 @@ +from photoflow.core import Store +from photoflow.core.models import Asset, Cluster, ClusterMember + + +def _seed(tmp_path): + s = Store(str(tmp_path / "t.db")).connect() + for i, t in [("a", "2019-06-01"), ("b", "2019-06-02"), ("c", "2019-06-03"), + ("d", "2019-07-01"), ("e", "2019-07-02")]: + s.upsert_asset(Asset(immich_id=i, taken_at=t)) + return s + + +def _members(ids): + return [ClusterMember(cluster_id=0, immich_id=i) for i in ids] + + +def test_insert_and_members_ordered(tmp_path): + s = _seed(tmp_path) + cid = s.insert_cluster(Cluster(start_at="2019-06-01", end_at="2019-06-03", + count=3, suggested_name="Trip 1", confidence=0.9), + _members(["c", "a", "b"])) + pairs = s.cluster_members(cid) + assert [a.immich_id for a, m in pairs] == ["a", "b", "c"] + assert s.get_cluster(cid).suggested_name == "Trip 1" + s.close() + + +def test_attention_sort(tmp_path): + s = _seed(tmp_path) + s.insert_cluster(Cluster(start_at="2019-06-01", end_at="2019-06-10", + confidence=0.9, status="pending"), _members(["a"])) + low = s.insert_cluster(Cluster(start_at="2019-07-01", end_at="2019-07-02", + confidence=0.2, status="pending"), _members(["d"])) + s.insert_cluster(Cluster(start_at="2019-06-02", end_at="2019-06-03", + confidence=0.1, status="approved"), _members(["b"])) + order = [c.id for c in s.clusters_by_attention()] + assert order[0] == low # lowest-confidence pending first + assert order[-1] != low # approved sinks to the bottom + s.close() + + +def test_chronological_neighbors(tmp_path): + s = _seed(tmp_path) + c1 = s.insert_cluster(Cluster(start_at="2019-06-01", end_at="2019-06-03"), _members(["a"])) + c2 = s.insert_cluster(Cluster(start_at="2019-07-01", end_at="2019-07-02"), _members(["d"])) + assert s.chronological_neighbors(c1) == (None, c2) + assert s.chronological_neighbors(c2) == (c1, None) + s.close() + + +def test_update_and_member_inclusion(tmp_path): + s = _seed(tmp_path) + cid = s.insert_cluster(Cluster(start_at="2019-06-01", end_at="2019-06-03"), + _members(["a", "b"])) + s.update_cluster(cid, status="approved", decided_name="Venice", reviewed_at="now") + c = s.get_cluster(cid) + assert c.status == "approved" and c.decided_name == "Venice" + s.set_member_inclusion(cid, "b", False) + inc = {m.immich_id: m.included for _, m in s.cluster_members(cid)} + assert inc == {"a": True, "b": False} + s.close() + + +def test_split_cluster(tmp_path): + s = _seed(tmp_path) + cid = s.insert_cluster(Cluster(start_at="2019-06-01", end_at="2019-06-03", + suggested_name="Trip"), _members(["a", "b", "c"])) + id1, id2 = s.split_cluster(cid, "c") # boundary "c" begins the second cluster + assert s.get_cluster(cid).status == "split" + left = [a.immich_id for a, _ in s.cluster_members(id1)] + right = [a.immich_id for a, _ in s.cluster_members(id2)] + assert left == ["a", "b"] and right == ["c"] + assert s.get_cluster(id1).status == "pending" + s.close() + + +def test_merge_clusters(tmp_path): + s = _seed(tmp_path) + a = s.insert_cluster(Cluster(start_at="2019-06-01", end_at="2019-06-03", + suggested_name="A"), _members(["a", "b"])) + b = s.insert_cluster(Cluster(start_at="2019-07-01", end_at="2019-07-02", + suggested_name="B"), _members(["d", "e"])) + new = s.merge_clusters(a, b) + assert s.get_cluster(a).status == "merged" and s.get_cluster(b).status == "merged" + ids = [x.immich_id for x, _ in s.cluster_members(new)] + assert ids == ["a", "b", "d", "e"] + c = s.get_cluster(new) + assert c.start_at == "2019-06-01" and c.end_at == "2019-07-02" and c.status == "pending" + s.close() + + +def test_writeback_log_idempotency(tmp_path): + s = _seed(tmp_path) + assert s.already_applied("a", "trip", "Venice") is False + s.log_writeback("a", "trip", "Venice", "ok") + assert s.already_applied("a", "trip", "Venice") is True + s.log_writeback("b", "trip", "Venice", "error:boom") + assert s.already_applied("b", "trip", "Venice") is False # only ok counts + s.close() -- 2.54.0 From e01b30a5b567750b05dd4e98bcedc2d97dc7e353 Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:19:06 +0200 Subject: [PATCH 09/19] feat(trip-cluster): config, factory, CLI skeleton, health + thumb proxy --- apps/trip-cluster/app/__init__.py | 21 +++++++++ apps/trip-cluster/app/cli.py | 60 ++++++++++++++++++++++++ apps/trip-cluster/app/config.py | 41 ++++++++++++++++ apps/trip-cluster/app/routes/__init__.py | 0 apps/trip-cluster/app/routes/nav.py | 13 +++++ apps/trip-cluster/app/routes/proxy.py | 14 ++++++ apps/trip-cluster/app/routes/review.py | 3 ++ apps/trip-cluster/tests/test_config.py | 18 +++++++ apps/trip-cluster/tests/test_routes.py | 26 ++++++++++ 9 files changed, 196 insertions(+) create mode 100644 apps/trip-cluster/app/cli.py create mode 100644 apps/trip-cluster/app/config.py create mode 100644 apps/trip-cluster/app/routes/__init__.py create mode 100644 apps/trip-cluster/app/routes/nav.py create mode 100644 apps/trip-cluster/app/routes/proxy.py create mode 100644 apps/trip-cluster/app/routes/review.py create mode 100644 apps/trip-cluster/tests/test_config.py create mode 100644 apps/trip-cluster/tests/test_routes.py diff --git a/apps/trip-cluster/app/__init__.py b/apps/trip-cluster/app/__init__.py index e69de29..08f3fa3 100644 --- a/apps/trip-cluster/app/__init__.py +++ b/apps/trip-cluster/app/__init__.py @@ -0,0 +1,21 @@ +from flask import Flask + +from app.config import load_config +from photoflow.ui import register_shared_ui + + +def create_app(config=None) -> Flask: + app = Flask(__name__) + cfg = config or load_config() + app.config["APP_CONFIG"] = cfg + app.config["DATA_DIR"] = cfg.data_dir + + register_shared_ui(app) + + from app.routes.nav import bp as nav_bp + from app.routes.review import bp as review_bp + from app.routes.proxy import bp as proxy_bp + app.register_blueprint(nav_bp) + app.register_blueprint(review_bp) + app.register_blueprint(proxy_bp) + return app diff --git a/apps/trip-cluster/app/cli.py b/apps/trip-cluster/app/cli.py new file mode 100644 index 0000000..f81c172 --- /dev/null +++ b/apps/trip-cluster/app/cli.py @@ -0,0 +1,60 @@ +import argparse +import sys + +from app.config import load_config + + +def _store(cfg): + import os + from photoflow.core import Store + os.makedirs(cfg.data_dir, exist_ok=True) + return Store(cfg.db_path).connect() + + +def _immich(deps, cfg): + if "immich" in deps: + return deps["immich"] + from photoflow.immich import ImmichClient + return ImmichClient(cfg.immich_url, cfg.immich_api_key) + + +def cmd_serve(deps) -> int: + from app import create_app + create_app(deps["config"]).run(host="0.0.0.0", port=8084) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="categorize") + sub = p.add_subparsers(dest="command", required=True) + + ing = sub.add_parser("ingest") + ing.add_argument("--from", dest="date_from") + ing.add_argument("--to", dest="date_to") + ing.add_argument("--tag") + ing.add_argument("--subset", type=int) + ing.add_argument("--full", action="store_true", help="ignore incremental updatedAfter") + + cl = sub.add_parser("cluster") + cl.add_argument("--gap-factor", type=float, default=6.0) + + sub.add_parser("serve") + + ap = sub.add_parser("apply") + ap.add_argument("--yes", action="store_true", help="skip the confirmation prompt") + + return p + + +def main(argv=None) -> int: + args = build_parser().parse_args(argv) + deps = {"config": load_config()} + if args.command == "serve": + return cmd_serve(deps) + # ingest / cluster / apply are wired in later tasks. + print(f"Command '{args.command}' is not implemented yet.") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/apps/trip-cluster/app/config.py b/apps/trip-cluster/app/config.py new file mode 100644 index 0000000..452504c --- /dev/null +++ b/apps/trip-cluster/app/config.py @@ -0,0 +1,41 @@ +import os +from dataclasses import dataclass +from typing import Mapping, Optional + +REQUIRED = ["IMMICH_URL", "IMMICH_API_KEY"] + + +class ConfigError(Exception): + def __init__(self, missing: list): + self.missing = missing + super().__init__(f"Missing required environment variables: {', '.join(missing)}") + + +@dataclass +class Config: + immich_url: str + immich_api_key: str + anthropic_api_key: str + data_dir: str + + @property + def db_path(self) -> str: + return os.path.join(self.data_dir, "trip-cluster.db") + + @property + def thumbs_dir(self) -> str: + return os.path.join(self.data_dir, "thumbs") + + +def load_config(env: Optional[Mapping] = None) -> Config: + env = env if env is not None else os.environ + missing = [k for k in REQUIRED if not (env.get(k) or "").strip()] + if missing: + raise ConfigError(missing) + data_dir = (env.get("DATA_DIR") or "").strip() or os.path.join(os.getcwd(), "data") + return Config( + immich_url=env["IMMICH_URL"].strip().rstrip("/"), + immich_api_key=env["IMMICH_API_KEY"].strip(), + anthropic_api_key=(env.get("ANTHROPIC_API_KEY") or "").strip(), + data_dir=data_dir, + ) diff --git a/apps/trip-cluster/app/routes/__init__.py b/apps/trip-cluster/app/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/apps/trip-cluster/app/routes/nav.py b/apps/trip-cluster/app/routes/nav.py new file mode 100644 index 0000000..2245db8 --- /dev/null +++ b/apps/trip-cluster/app/routes/nav.py @@ -0,0 +1,13 @@ +from flask import Blueprint + +bp = Blueprint("nav", __name__) + + +@bp.route("/health") +def health(): + return "ok" + + +@bp.route("/") +def index(): + return "trip-cluster" diff --git a/apps/trip-cluster/app/routes/proxy.py b/apps/trip-cluster/app/routes/proxy.py new file mode 100644 index 0000000..302e85c --- /dev/null +++ b/apps/trip-cluster/app/routes/proxy.py @@ -0,0 +1,14 @@ +import os +from flask import Blueprint, current_app, send_file, abort + +bp = Blueprint("proxy", __name__) + + +@bp.route("/thumb/") +def thumb(asset_id): + cfg = current_app.config["APP_CONFIG"] + safe = os.path.basename(asset_id) + path = os.path.join(cfg.thumbs_dir, f"{safe}.jpg") + if not os.path.exists(path): + abort(404) + return send_file(path, mimetype="image/jpeg") diff --git a/apps/trip-cluster/app/routes/review.py b/apps/trip-cluster/app/routes/review.py new file mode 100644 index 0000000..de13b75 --- /dev/null +++ b/apps/trip-cluster/app/routes/review.py @@ -0,0 +1,3 @@ +from flask import Blueprint + +bp = Blueprint("review", __name__) diff --git a/apps/trip-cluster/tests/test_config.py b/apps/trip-cluster/tests/test_config.py new file mode 100644 index 0000000..523280e --- /dev/null +++ b/apps/trip-cluster/tests/test_config.py @@ -0,0 +1,18 @@ +import os +import pytest +from app.config import load_config, ConfigError + + +def test_missing_required_raises(): + with pytest.raises(ConfigError) as e: + load_config({"IMMICH_URL": "http://x"}) + assert "IMMICH_API_KEY" in e.value.missing + + +def test_anthropic_optional_and_paths(tmp_path): + cfg = load_config({"IMMICH_URL": "http://x/", "IMMICH_API_KEY": "k", + "DATA_DIR": str(tmp_path)}) + assert cfg.immich_url == "http://x" # trailing slash stripped + assert cfg.anthropic_api_key == "" # optional in M1 + assert cfg.db_path == os.path.join(str(tmp_path), "trip-cluster.db") + assert cfg.thumbs_dir == os.path.join(str(tmp_path), "thumbs") diff --git a/apps/trip-cluster/tests/test_routes.py b/apps/trip-cluster/tests/test_routes.py new file mode 100644 index 0000000..18de583 --- /dev/null +++ b/apps/trip-cluster/tests/test_routes.py @@ -0,0 +1,26 @@ +import os +from app import create_app +from app.config import Config + + +def _app(tmp_path): + cfg = Config(immich_url="http://x", immich_api_key="k", + anthropic_api_key="", data_dir=str(tmp_path)) + app = create_app(cfg) + app.config.update(TESTING=True) + return app + + +def test_health(tmp_path): + assert _app(tmp_path).test_client().get("/health").data == b"ok" + + +def test_thumb_served(tmp_path): + thumbs = os.path.join(str(tmp_path), "thumbs") + os.makedirs(thumbs, exist_ok=True) + with open(os.path.join(thumbs, "a.jpg"), "wb") as f: + f.write(b"\xff\xd8\xffjpeg") + client = _app(tmp_path).test_client() + r = client.get("/thumb/a") + assert r.status_code == 200 and r.mimetype == "image/jpeg" + assert _app(tmp_path).test_client().get("/thumb/missing").status_code == 404 -- 2.54.0 From 8d58ec03a99d0efb48fb392838ca5a9f797bc96d Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:19:06 +0200 Subject: [PATCH 10/19] =?UTF-8?q?feat(trip-cluster):=20coverage=20detectio?= =?UTF-8?q?n=20=E2=80=94=20completeness=20flags=20+=20outliers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/trip-cluster/app/coverage.py | 35 +++++++++++++++++ apps/trip-cluster/tests/test_coverage.py | 48 ++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 apps/trip-cluster/app/coverage.py create mode 100644 apps/trip-cluster/tests/test_coverage.py diff --git a/apps/trip-cluster/app/coverage.py b/apps/trip-cluster/app/coverage.py new file mode 100644 index 0000000..f160c6e --- /dev/null +++ b/apps/trip-cluster/app/coverage.py @@ -0,0 +1,35 @@ +from photoflow.core.models import ClusterMember +from app.clustering import _epoch, _median + + +def coverage_members(candidate, assets_by_id, *, outlier_factor: float = 8.0) -> list: + member_ids = [i for i in candidate.member_ids if i in assets_by_id] + members = sorted((assets_by_id[i] for i in member_ids), key=lambda a: a.taken_at) + epochs = [_epoch(a.taken_at) for a in members] + gaps = [b - a for a, b in zip(epochs, epochs[1:])] + base = _median(gaps) if gaps else 0.0 + + out = [] + for idx, a in enumerate(members): + is_outlier = False + if candidate.seed_tag and base > 0: + left = epochs[idx] - epochs[idx - 1] if idx > 0 else 0 + right = epochs[idx + 1] - epochs[idx] if idx < len(members) - 1 else 0 + nearest = min([g for g in (left, right) if g > 0], default=0) + if nearest > outlier_factor * base: + is_outlier = True + out.append(ClusterMember( + cluster_id=0, immich_id=a.immich_id, + member_confidence=candidate.confidence, is_outlier=is_outlier, + included=True, flagged_coverage=False)) + + if candidate.seed_tag and candidate.start_at and candidate.end_at: + mset = set(member_ids) + for a in sorted(assets_by_id.values(), key=lambda x: x.taken_at): + if a.immich_id in mset: + continue + if candidate.start_at <= a.taken_at <= candidate.end_at: + out.append(ClusterMember( + cluster_id=0, immich_id=a.immich_id, member_confidence=0.0, + is_outlier=False, included=False, flagged_coverage=True)) + return out diff --git a/apps/trip-cluster/tests/test_coverage.py b/apps/trip-cluster/tests/test_coverage.py new file mode 100644 index 0000000..6aa4f8d --- /dev/null +++ b/apps/trip-cluster/tests/test_coverage.py @@ -0,0 +1,48 @@ +from photoflow.core.models import Asset +from app.clustering import CandidateCluster +from app.coverage import coverage_members + + +def _a(i, taken): + return Asset(immich_id=i, taken_at=taken) + + +def test_coverage_candidate_inside_seeded_window(): + members = [_a("a", "2019-06-01T10:00:00"), _a("b", "2019-06-01T12:00:00"), + _a("c", "2019-06-02T10:00:00")] + intruder = _a("x", "2019-06-01T13:00:00") # in window, untagged + outside = _a("y", "2019-07-01T10:00:00") # out of window + by_id = {m.immich_id: m for m in members + [intruder, outside]} + cand = CandidateCluster(member_ids=["a", "b", "c"], start_at="2019-06-01T10:00:00", + end_at="2019-06-02T10:00:00", suggested_name="Italy 2019", + confidence=0.95, kind_guess="trip", seed_tag="Italy 2019") + out = coverage_members(cand, by_id) + flagged = {m.immich_id for m in out if m.flagged_coverage} + assert flagged == {"x"} # only the in-window intruder + assert all(not m.included for m in out if m.flagged_coverage) + + +def test_outlier_member_far_from_bulk(): + members = [_a("a", "2019-06-01T10:00:00"), _a("a2", "2019-06-01T11:00:00"), + _a("a3", "2019-06-01T12:00:00"), + _a("z", "2019-09-01T10:00:00")] # tagged but months away + by_id = {m.immich_id: m for m in members} + cand = CandidateCluster(member_ids=["a", "a2", "a3", "z"], + start_at="2019-06-01T10:00:00", end_at="2019-09-01T10:00:00", + suggested_name="Italy 2019", confidence=0.95, + kind_guess="trip", seed_tag="Italy 2019") + out = coverage_members(cand, by_id) + outliers = {m.immich_id for m in out if m.is_outlier} + assert outliers == {"z"} + + +def test_non_seeded_cluster_has_no_coverage_or_outliers(): + members = [_a("a", "2019-06-01T10:00:00"), _a("b", "2019-06-01T12:00:00")] + intruder = _a("x", "2019-06-01T11:00:00") + by_id = {m.immich_id: m for m in members + [intruder]} + cand = CandidateCluster(member_ids=["a", "b"], start_at="2019-06-01T10:00:00", + end_at="2019-06-01T12:00:00", suggested_name="Trip", + confidence=0.4, kind_guess="everyday", seed_tag=None) + out = coverage_members(cand, by_id) + assert {m.immich_id for m in out} == {"a", "b"} + assert not any(m.flagged_coverage or m.is_outlier for m in out) -- 2.54.0 From 6d0c9662eede573c050b64bdf430ce13257c160f Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:21:38 +0200 Subject: [PATCH 11/19] =?UTF-8?q?feat(trip-cluster):=20ingest=20=E2=80=94?= =?UTF-8?q?=20scopeable,=20incremental,=20processed=20read-back?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/trip-cluster/app/cli.py | 18 ++++- apps/trip-cluster/app/ingest.py | 80 +++++++++++++++++++ apps/trip-cluster/tests/test_ingest.py | 104 +++++++++++++++++++++++++ 3 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 apps/trip-cluster/app/ingest.py create mode 100644 apps/trip-cluster/tests/test_ingest.py diff --git a/apps/trip-cluster/app/cli.py b/apps/trip-cluster/app/cli.py index f81c172..956a55d 100644 --- a/apps/trip-cluster/app/cli.py +++ b/apps/trip-cluster/app/cli.py @@ -18,6 +18,19 @@ def _immich(deps, cfg): return ImmichClient(cfg.immich_url, cfg.immich_api_key) +def cmd_ingest(deps, *, date_from, date_to, tag, subset, full) -> int: + cfg = deps["config"] + store = _store(cfg) + client = _immich(deps, cfg) + from app.ingest import run_ingest + res = run_ingest(client, store, cfg.thumbs_dir, date_from=date_from, + date_to=date_to, tag=tag, subset=subset, full=full) + store.close() + print(f"Ingested {res['fetched']} asset(s); " + f"{res['processed_marked']} already-processed.") + return 0 + + def cmd_serve(deps) -> int: from app import create_app create_app(deps["config"]).run(host="0.0.0.0", port=8084) @@ -51,7 +64,10 @@ def main(argv=None) -> int: deps = {"config": load_config()} if args.command == "serve": return cmd_serve(deps) - # ingest / cluster / apply are wired in later tasks. + if args.command == "ingest": + return cmd_ingest(deps, date_from=args.date_from, date_to=args.date_to, + tag=args.tag, subset=args.subset, full=args.full) + # cluster / apply are wired in later tasks. print(f"Command '{args.command}' is not implemented yet.") return 1 diff --git a/apps/trip-cluster/app/ingest.py b/apps/trip-cluster/app/ingest.py new file mode 100644 index 0000000..d207ad0 --- /dev/null +++ b/apps/trip-cluster/app/ingest.py @@ -0,0 +1,80 @@ +import datetime +import logging +import os + +from photoflow.core.models import Asset +from photoflow.immich import pipeline + +log = logging.getLogger(__name__) + + +def _now() -> str: + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def run_ingest(client, store, thumbs_dir, *, date_from=None, date_to=None, + tag=None, subset=None, full=False) -> dict: + os.makedirs(thumbs_dir, exist_ok=True) + + tag_ids = None + if tag: + tid = client.resolve_tag_id(tag) + if not tid: + raise ValueError(f"Tag {tag!r} not found in Immich") + tag_ids = [tid] + + updated_after = None if full else store.get_meta("last_ingest_at") + + assets = client.search_assets(taken_after=date_from, taken_before=date_to, + tag_ids=tag_ids, updated_after=updated_after) + if subset is not None: + assets = assets[:subset] + + processed_marked = 0 + max_updated = updated_after or "" + tag_counts: dict = {} + + for a in assets: + # Revision (2): sanitize the Immich-provided id before using it as a + # path component, guarding against path traversal (mirror /thumb proxy). + safe_id = os.path.basename(a["id"]) + thumb_path = os.path.join(thumbs_dir, f"{safe_id}.jpg") + # Revision (1): treat an existing 0-byte file as missing. + need_thumb = (not os.path.exists(thumb_path) + or os.path.getsize(thumb_path) == 0) + if need_thumb: + try: + data = client.download_thumbnail(a["id"]) + # Write to a temp path and os.replace into place so a failure + # never leaves a 0-byte .jpg behind. + tmp_path = f"{thumb_path}.tmp" + with open(tmp_path, "wb") as f: + f.write(data) + os.replace(tmp_path, thumb_path) + except Exception: + log.exception("Thumbnail download failed for asset %s", a["id"]) + continue + + store.upsert_asset(Asset( + immich_id=a["id"], taken_at=a["taken_at"], gps_lat=a["gps_lat"], + gps_lon=a["gps_lon"], place_city=a["place_city"], + place_country=a["place_country"], type=a["type"], + thumb_path=thumb_path, ingested_at=_now(), updated_at=a["updated_at"])) + store.set_asset_tags(a["id"], a["tags"]) + for t in a["tags"]: + tag_counts[t] = tag_counts.get(t, 0) + 1 + + if pipeline.PROCESSED in a["tags"]: + store.mark_processed(a["id"]) + processed_marked += 1 + + if a["updated_at"] and a["updated_at"] > max_updated: + max_updated = a["updated_at"] + + for name, count in tag_counts.items(): + store.upsert_tag(name, count=count) + + if max_updated: + store.set_meta("last_ingest_at", max_updated) + + return {"fetched": len(assets), "processed_marked": processed_marked} diff --git a/apps/trip-cluster/tests/test_ingest.py b/apps/trip-cluster/tests/test_ingest.py new file mode 100644 index 0000000..25896b8 --- /dev/null +++ b/apps/trip-cluster/tests/test_ingest.py @@ -0,0 +1,104 @@ +import os +from photoflow.core import Store +from photoflow.immich import pipeline +from app.ingest import run_ingest + + +class FakeImmich: + def __init__(self, assets, tag_map=None, thumb=b"\xff\xd8\xffjpeg"): + self._assets = assets + self._tag_map = tag_map or {} + self._thumb = thumb + self.searches = [] + + def resolve_tag_id(self, name): + return self._tag_map.get(name) + + def search_assets(self, **kwargs): + self.searches.append(kwargs) + return list(self._assets) + + def download_thumbnail(self, asset_id): + return self._thumb + + +def _asset(i, taken, tags=None, updated="2026-01-01T00:00:00Z"): + return {"id": i, "original_filename": f"{i}.jpg", "taken_at": taken, + "gps_lat": None, "gps_lon": None, "place_city": None, + "place_country": None, "type": "IMAGE", "tags": tags or [], + "rating": 0, "updated_at": updated} + + +def test_ingest_upserts_assets_tags_and_thumbs(tmp_path): + store = Store(str(tmp_path / "t.db")).connect() + thumbs = str(tmp_path / "thumbs") + os.makedirs(thumbs, exist_ok=True) + client = FakeImmich([_asset("a", "2019-06-01", tags=["Italy 2019"]), + _asset("b", "2019-06-02")]) + res = run_ingest(client, store, thumbs, date_from="2019-01-01", date_to="2020-01-01") + assert res["fetched"] == 2 + assert store.get_asset("a").taken_at == "2019-06-01" + assert store.asset_tags("a") == ["Italy 2019"] + assert os.path.exists(os.path.join(thumbs, "a.jpg")) + assert store.get_meta("last_ingest_at") == "2026-01-01T00:00:00Z" + store.close() + + +def test_ingest_marks_processed_from_pipeline_tag(tmp_path): + store = Store(str(tmp_path / "t.db")).connect() + thumbs = str(tmp_path / "thumbs"); os.makedirs(thumbs, exist_ok=True) + client = FakeImmich([_asset("a", "2019-06-01", tags=[pipeline.PROCESSED])]) + res = run_ingest(client, store, thumbs) + assert res["processed_marked"] == 1 + assert store.get_asset("a").processed is True + store.close() + + +def test_ingest_incremental_passes_updated_after(tmp_path): + store = Store(str(tmp_path / "t.db")).connect() + thumbs = str(tmp_path / "thumbs"); os.makedirs(thumbs, exist_ok=True) + store.set_meta("last_ingest_at", "2026-05-01T00:00:00Z") + client = FakeImmich([_asset("a", "2019-06-01")]) + run_ingest(client, store, thumbs) + assert client.searches[0].get("updated_after") == "2026-05-01T00:00:00Z" + # --full ignores it + client2 = FakeImmich([_asset("a", "2019-06-01")]) + run_ingest(client2, store, thumbs, full=True) + assert client2.searches[0].get("updated_after") is None + store.close() + + +def test_ingest_subset_and_tag_resolution(tmp_path): + store = Store(str(tmp_path / "t.db")).connect() + thumbs = str(tmp_path / "thumbs"); os.makedirs(thumbs, exist_ok=True) + client = FakeImmich([_asset("a", "2019-06-01"), _asset("b", "2019-06-02")], + tag_map={"Italy 2019": "t1"}) + res = run_ingest(client, store, thumbs, tag="Italy 2019", subset=1) + assert res["fetched"] == 1 + assert client.searches[0].get("tag_ids") == ["t1"] + store.close() + + +def test_ingest_resilient_to_thumb_failure_and_zero_byte(tmp_path): + store = Store(str(tmp_path / "t.db")).connect() + thumbs = str(tmp_path / "thumbs"); os.makedirs(thumbs, exist_ok=True) + + class FailThenOk(FakeImmich): + def download_thumbnail(self, asset_id): + if asset_id == "a": + raise RuntimeError("boom") + return self._thumb + + # Pre-existing 0-byte file for "b" must be treated as missing and refetched. + with open(os.path.join(thumbs, "b.jpg"), "wb"): + pass + + client = FailThenOk([_asset("a", "2019-06-01"), _asset("b", "2019-06-02")]) + res = run_ingest(client, store, thumbs) + # The run completes despite "a" failing. + assert res["fetched"] == 2 + # No 0-byte file left for the failed asset. + assert not os.path.exists(os.path.join(thumbs, "a.jpg")) + # The 0-byte file for "b" was replaced with real bytes. + assert os.path.getsize(os.path.join(thumbs, "b.jpg")) > 0 + store.close() -- 2.54.0 From a9b443cd53c9df02c30511d8b3d5699526738e15 Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:21:38 +0200 Subject: [PATCH 12/19] feat(trip-cluster): cluster review operations (approve/non-trip/skip/split/merge/bulk) --- apps/trip-cluster/app/review.py | 41 ++++++++++++++++++ apps/trip-cluster/tests/test_review.py | 59 ++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 apps/trip-cluster/app/review.py create mode 100644 apps/trip-cluster/tests/test_review.py diff --git a/apps/trip-cluster/app/review.py b/apps/trip-cluster/app/review.py new file mode 100644 index 0000000..2c2e76b --- /dev/null +++ b/apps/trip-cluster/app/review.py @@ -0,0 +1,41 @@ +import datetime + + +def _now() -> str: + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def approve(store, cluster_id, name=None) -> None: + c = store.get_cluster(cluster_id) + decided = (name or "").strip() or c.suggested_name + store.update_cluster(cluster_id, status="approved", + decided_name=decided, reviewed_at=_now()) + + +def mark_non_trip(store, cluster_id) -> None: + store.update_cluster(cluster_id, status="non_trip", reviewed_at=_now()) + + +def skip(store, cluster_id) -> None: + store.update_cluster(cluster_id, status="skipped", reviewed_at=_now()) + + +def split(store, cluster_id, boundary_immich_id): + return store.split_cluster(cluster_id, boundary_immich_id) + + +def merge(store, cluster_id_a, cluster_id_b): + return store.merge_clusters(cluster_id_a, cluster_id_b) + + +def set_member(store, cluster_id, immich_id, included: bool) -> None: + store.set_member_inclusion(cluster_id, immich_id, included) + + +def approve_high_confidence(store, threshold: float = 0.75) -> int: + count = 0 + for c in store.all_clusters(): + if c.status == "pending" and c.kind_guess == "trip" and c.confidence >= threshold: + approve(store, c.id) + count += 1 + return count diff --git a/apps/trip-cluster/tests/test_review.py b/apps/trip-cluster/tests/test_review.py new file mode 100644 index 0000000..8c25208 --- /dev/null +++ b/apps/trip-cluster/tests/test_review.py @@ -0,0 +1,59 @@ +from photoflow.core import Store +from photoflow.core.models import Asset, Cluster, ClusterMember +from app import review + + +def _store(tmp_path): + s = Store(str(tmp_path / "t.db")).connect() + for i, t in [("a", "2019-06-01"), ("b", "2019-06-02"), ("c", "2019-06-03")]: + s.upsert_asset(Asset(immich_id=i, taken_at=t)) + return s + + +def _cluster(s, name="Trip", conf=0.9, kind="trip", ids=("a", "b")): + return s.insert_cluster( + Cluster(start_at="2019-06-01", end_at="2019-06-03", suggested_name=name, + confidence=conf, kind_guess=kind, status="pending"), + [ClusterMember(cluster_id=0, immich_id=i) for i in ids]) + + +def test_approve_uses_suggested_when_no_name(tmp_path): + s = _store(tmp_path) + cid = _cluster(s, name="Venice") + review.approve(s, cid) + c = s.get_cluster(cid) + assert c.status == "approved" and c.decided_name == "Venice" and c.reviewed_at + review.approve(s, cid, name="Venezia") + assert s.get_cluster(cid).decided_name == "Venezia" + s.close() + + +def test_non_trip_skip(tmp_path): + s = _store(tmp_path) + cid = _cluster(s) + review.mark_non_trip(s, cid) + assert s.get_cluster(cid).status == "non_trip" + c2 = _cluster(s) + review.skip(s, c2) + assert s.get_cluster(c2).status == "skipped" + s.close() + + +def test_set_member(tmp_path): + s = _store(tmp_path) + cid = _cluster(s, ids=("a", "b")) + review.set_member(s, cid, "b", False) + inc = {m.immich_id: m.included for _, m in s.cluster_members(cid)} + assert inc["b"] is False + s.close() + + +def test_approve_high_confidence_only_trips(tmp_path): + s = _store(tmp_path) + hi = _cluster(s, conf=0.9, kind="trip") + _cluster(s, conf=0.3, kind="trip") # too low + _cluster(s, conf=0.9, kind="everyday") # everyday excluded + n = review.approve_high_confidence(s, threshold=0.75) + assert n == 1 + assert s.get_cluster(hi).status == "approved" + s.close() -- 2.54.0 From 5256df4117b4ef76fdc36fd16e4d8d00b6ab3065 Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:28:27 +0200 Subject: [PATCH 13/19] feat(trip-cluster): cluster orchestration (seed derivation + coverage) + CLI --- apps/trip-cluster/app/cli.py | 14 ++++- apps/trip-cluster/app/cluster_run.py | 57 +++++++++++++++++++ apps/trip-cluster/tests/test_cluster_run.py | 61 +++++++++++++++++++++ 3 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 apps/trip-cluster/app/cluster_run.py create mode 100644 apps/trip-cluster/tests/test_cluster_run.py diff --git a/apps/trip-cluster/app/cli.py b/apps/trip-cluster/app/cli.py index 956a55d..4bb7023 100644 --- a/apps/trip-cluster/app/cli.py +++ b/apps/trip-cluster/app/cli.py @@ -31,6 +31,16 @@ def cmd_ingest(deps, *, date_from, date_to, tag, subset, full) -> int: return 0 +def cmd_cluster(deps, *, gap_factor) -> int: + cfg = deps["config"] + store = _store(cfg) + from app.cluster_run import run_cluster + res = run_cluster(store, gap_factor=gap_factor) + store.close() + print(f"Built {res['clusters']} candidate cluster(s). Run 'categorize serve' to review.") + return 0 + + def cmd_serve(deps) -> int: from app import create_app create_app(deps["config"]).run(host="0.0.0.0", port=8084) @@ -67,7 +77,9 @@ def main(argv=None) -> int: if args.command == "ingest": return cmd_ingest(deps, date_from=args.date_from, date_to=args.date_to, tag=args.tag, subset=args.subset, full=args.full) - # cluster / apply are wired in later tasks. + if args.command == "cluster": + return cmd_cluster(deps, gap_factor=args.gap_factor) + # apply is wired in a later task. print(f"Command '{args.command}' is not implemented yet.") return 1 diff --git a/apps/trip-cluster/app/cluster_run.py b/apps/trip-cluster/app/cluster_run.py new file mode 100644 index 0000000..eb58ea6 --- /dev/null +++ b/apps/trip-cluster/app/cluster_run.py @@ -0,0 +1,57 @@ +from collections import defaultdict + +from photoflow.core.models import Cluster +from photoflow.immich import pipeline +from app.clustering import cluster_assets, _epoch +from app.coverage import coverage_members + +# A tag seeds only if its photos dominate the time window they span: among all +# assets whose taken_at falls in [min, max] of the tag's photos, at least this +# fraction must carry the tag. A name/event tag sprinkled among many untagged +# photos in the same window fails this gate. +SEED_DOMINANCE_MIN = 0.5 + + +def _seed_tags(assets, tags_by_asset, max_span_days: int, + dominance_min: float = SEED_DOMINANCE_MIN) -> set: + epochs = {a.immich_id: _epoch(a.taken_at) for a in assets} + tagged = defaultdict(list) # tag -> epochs of its assets + for a in assets: + for t in tags_by_asset.get(a.immich_id, []): + if not pipeline.is_pipeline_tag(t): + tagged[t].append(epochs[a.immich_id]) + + seeds = set() + for tag, ts in tagged.items(): + if len(ts) < 2: + continue + lo, hi = min(ts), max(ts) + # Loose safety cap only — reject absurdly long spans, but keep genuine + # long trips (this library has one > 60 days), so the default is high. + if hi - lo > max_span_days * 86400: + continue + # Dominance / temporal contiguity gate. + in_window = sum(1 for e in epochs.values() if lo <= e <= hi) + if in_window and len(ts) / in_window >= dominance_min: + seeds.add(tag) + return seeds + + +def run_cluster(store, *, gap_factor: float = 6.0, seed_max_span_days: int = 400) -> dict: + assets = store.all_assets(include_processed=False) + tags_by_asset = {a.immich_id: store.asset_tags(a.immich_id) for a in assets} + seed_tags = _seed_tags(assets, tags_by_asset, seed_max_span_days) + candidates = cluster_assets(assets, tags_by_asset, seed_tags, gap_factor=gap_factor) + assets_by_id = {a.immich_id: a for a in assets} + + store.clear_clusters() + n = 0 + for cand in candidates: + members = coverage_members(cand, assets_by_id) + store.insert_cluster(Cluster( + start_at=cand.start_at, end_at=cand.end_at, + count=sum(1 for m in members if m.included), + suggested_name=cand.suggested_name, confidence=cand.confidence, + kind_guess=cand.kind_guess, status="pending"), members) + n += 1 + return {"clusters": n} diff --git a/apps/trip-cluster/tests/test_cluster_run.py b/apps/trip-cluster/tests/test_cluster_run.py new file mode 100644 index 0000000..ccc38db --- /dev/null +++ b/apps/trip-cluster/tests/test_cluster_run.py @@ -0,0 +1,61 @@ +from photoflow.core import Store +from photoflow.core.models import Asset +from photoflow.immich import pipeline +from app.cluster_run import run_cluster + + +def _store(tmp_path): + return Store(str(tmp_path / "t.db")).connect() + + +def test_run_cluster_seeds_bounded_tag_not_people_tag(tmp_path): + s = _store(tmp_path) + # "Italy 2019" spans 2 days -> seeds; "Mum" spans years -> not a seed. + rows = [("a", "2019-06-01T10:00:00", ["Italy 2019", "Mum"]), + ("b", "2019-06-02T10:00:00", ["Italy 2019"]), + ("c", "2010-01-01T10:00:00", ["Mum"]), + ("d", "2022-01-01T10:00:00", ["Mum"])] + for i, t, tags in rows: + s.upsert_asset(Asset(immich_id=i, taken_at=t)) + s.set_asset_tags(i, tags) + res = run_cluster(s) + seeded = [c for c in s.all_clusters() if c.suggested_name == "Italy 2019"] + assert len(seeded) == 1 + assert sorted(x.immich_id for x, _ in s.cluster_members(seeded[0].id)) == ["a", "b"] + assert res["clusters"] >= 1 + s.close() + + +def test_run_cluster_excludes_processed_and_replaces(tmp_path): + s = _store(tmp_path) + s.upsert_asset(Asset(immich_id="a", taken_at="2019-06-01T10:00:00")) + s.upsert_asset(Asset(immich_id="p", taken_at="2019-06-01T11:00:00")) + s.set_asset_tags("a", []) + s.set_asset_tags("p", [pipeline.PROCESSED]) + s.mark_processed("p") + run_cluster(s) + run_cluster(s) # idempotent replace — not doubled + all_member_ids = [x.immich_id for c in s.all_clusters() + for x, _ in s.cluster_members(c.id)] + assert "p" not in all_member_ids + assert all_member_ids.count("a") == 1 + s.close() + + +def test_run_cluster_short_span_non_dominant_tag_does_not_seed(tmp_path): + s = _store(tmp_path) + # "Lunch" is on 2 photos within a ~30-min window, but that same window is + # full of untagged photos -> the tag does NOT dominate its window + # (2/5 = 0.4 < 0.5), so it must not seed even though its span is tiny. + rows = [("x", "2020-03-01T12:00:00", ["Lunch"]), + ("y", "2020-03-01T12:30:00", ["Lunch"]), + ("u1", "2020-03-01T12:05:00", []), + ("u2", "2020-03-01T12:15:00", []), + ("u3", "2020-03-01T12:25:00", [])] + for i, t, tags in rows: + s.upsert_asset(Asset(immich_id=i, taken_at=t)) + s.set_asset_tags(i, tags) + run_cluster(s) + seeded = [c for c in s.all_clusters() if c.suggested_name == "Lunch"] + assert seeded == [] + s.close() -- 2.54.0 From 44c05444d3fa94f8833311b90d3ab165377355b9 Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:35:58 +0200 Subject: [PATCH 14/19] feat(trip-cluster): idempotent write-back + CLI apply with confirmation --- apps/trip-cluster/app/cli.py | 27 +++++- apps/trip-cluster/app/writeback.py | 57 +++++++++++ apps/trip-cluster/tests/test_writeback.py | 113 ++++++++++++++++++++++ 3 files changed, 196 insertions(+), 1 deletion(-) create mode 100644 apps/trip-cluster/app/writeback.py create mode 100644 apps/trip-cluster/tests/test_writeback.py diff --git a/apps/trip-cluster/app/cli.py b/apps/trip-cluster/app/cli.py index 4bb7023..c87109f 100644 --- a/apps/trip-cluster/app/cli.py +++ b/apps/trip-cluster/app/cli.py @@ -41,6 +41,30 @@ def cmd_cluster(deps, *, gap_factor) -> int: return 0 +def cmd_apply(deps, *, yes) -> int: + cfg = deps["config"] + store = _store(cfg) + from app.writeback import apply_all, APPLYABLE + pending = [c for c in store.all_clusters() if c.status in APPLYABLE] + if not pending: + print("Nothing to apply.") + store.close() + return 0 + if not yes: + ans = input(f"Apply {len(pending)} cluster decision(s) to Immich? [y/N] ") + if ans.strip().lower() not in ("y", "yes"): + print("Aborted.") + store.close() + return 1 + client = _immich(deps, cfg) + results = apply_all(client, store) + store.close() + ok = sum(len(r["succeeded"]) for r in results) + bad = sum(len(r["failed"]) for r in results) + print(f"Applied {ok} tag write(s); {bad} failure(s) across {len(results)} cluster(s).") + return 0 if bad == 0 else 1 + + def cmd_serve(deps) -> int: from app import create_app create_app(deps["config"]).run(host="0.0.0.0", port=8084) @@ -79,7 +103,8 @@ def main(argv=None) -> int: tag=args.tag, subset=args.subset, full=args.full) if args.command == "cluster": return cmd_cluster(deps, gap_factor=args.gap_factor) - # apply is wired in a later task. + if args.command == "apply": + return cmd_apply(deps, yes=args.yes) print(f"Command '{args.command}' is not implemented yet.") return 1 diff --git a/apps/trip-cluster/app/writeback.py b/apps/trip-cluster/app/writeback.py new file mode 100644 index 0000000..7416583 --- /dev/null +++ b/apps/trip-cluster/app/writeback.py @@ -0,0 +1,57 @@ +from photoflow.immich import pipeline + +APPLYABLE = ("approved", "non_trip", "skipped") + + +def _apply_tag(client, store, asset_ids, action, tag, tag_id): + todo = [a for a in asset_ids if not store.already_applied(a, action, tag)] + if not todo: + return [], [] + try: + client.tag_assets(tag_id, todo) + except Exception as e: # noqa: BLE001 — recorded, surfaced + for a in todo: + store.log_writeback(a, action, tag, f"error:{e}") + return [], [(a, str(e)) for a in todo] + for a in todo: + store.log_writeback(a, action, tag, "ok") + return todo, [] + + +def apply_cluster(client, store, cluster_id) -> dict: + c = store.get_cluster(cluster_id) + if c is None or c.status not in APPLYABLE: + return {"cluster_id": cluster_id, "status": c.status if c else None, + "succeeded": [], "failed": []} + + included = [a.immich_id for a, m in store.cluster_members(cluster_id) if m.included] + succeeded, failed = [], [] + + if c.status == "approved": + tag = c.decided_name or c.suggested_name + ok, fail = _apply_tag(client, store, included, "trip", tag, client.upsert_tag(tag)) + succeeded += ok + failed += fail + elif c.status == "non_trip": + ok, fail = _apply_tag(client, store, included, "non-trip", pipeline.NON_TRIP, + client.upsert_tag(pipeline.NON_TRIP)) + succeeded += ok + failed += fail + # 'skipped': no content/non-trip tag, only processed below. + + failed_ids = {i for i, _ in failed} + proc_targets = [a for a in included if a not in failed_ids] + if proc_targets: + _apply_tag(client, store, proc_targets, "processed", pipeline.PROCESSED, + client.upsert_tag(pipeline.PROCESSED)) + for a in proc_targets: + if store.already_applied(a, "processed", pipeline.PROCESSED): + store.mark_processed(a) + + return {"cluster_id": cluster_id, "status": c.status, + "succeeded": succeeded, "failed": failed} + + +def apply_all(client, store) -> list: + return [apply_cluster(client, store, c.id) + for c in store.all_clusters() if c.status in APPLYABLE] diff --git a/apps/trip-cluster/tests/test_writeback.py b/apps/trip-cluster/tests/test_writeback.py new file mode 100644 index 0000000..9deb8d6 --- /dev/null +++ b/apps/trip-cluster/tests/test_writeback.py @@ -0,0 +1,113 @@ +from photoflow.core import Store +from photoflow.core.models import Asset, Cluster, ClusterMember +from photoflow.immich import pipeline +from app.writeback import apply_cluster, apply_all + + +class FakeImmich: + def __init__(self): + self.tagged = [] + self.fail_tag_id = None + self._ids = {} + + def upsert_tag(self, name): + self._ids.setdefault(name, f"id:{name}") + return self._ids[name] + + def tag_assets(self, tag_id, ids): + if self.fail_tag_id is not None and tag_id == self.fail_tag_id: + raise RuntimeError("boom") + self.tagged.append((tag_id, list(ids))) + + +def _store(tmp_path): + s = Store(str(tmp_path / "t.db")).connect() + for i in ("a", "b", "x"): + s.upsert_asset(Asset(immich_id=i, taken_at="2019-06-01")) + return s + + +def _approved(s, name="Venice"): + cid = s.insert_cluster( + Cluster(start_at="2019-06-01", end_at="2019-06-02", suggested_name=name, + status="approved", decided_name=name), + [ClusterMember(cluster_id=0, immich_id="a"), + ClusterMember(cluster_id=0, immich_id="b"), + ClusterMember(cluster_id=0, immich_id="x", included=False, flagged_coverage=True)]) + return cid + + +def test_apply_approved_tags_included_then_processed(tmp_path): + s = _store(tmp_path) + cid = _approved(s) + client = FakeImmich() + res = apply_cluster(client, s, cid) + assert sorted(res["succeeded"]) == ["a", "b"] and res["failed"] == [] + # trip tag on a,b ; processed on a,b ; x (excluded) never tagged + assert ("id:Venice", ["a", "b"]) in client.tagged + assert ("id:_pipeline/processed", ["a", "b"]) in client.tagged + assert all("x" not in ids for _, ids in client.tagged) + assert s.get_asset("a").processed is True + s.close() + + +def test_apply_is_idempotent(tmp_path): + s = _store(tmp_path) + cid = _approved(s) + client = FakeImmich() + apply_cluster(client, s, cid) + before = len(client.tagged) + apply_cluster(client, s, cid) # second run writes nothing new + assert len(client.tagged) == before + s.close() + + +def test_partial_failure_leaves_retryable(tmp_path): + s = _store(tmp_path) + cid = _approved(s) + client = FakeImmich() + client.fail_tag_id = "id:Venice" # trip tag write fails + res = apply_cluster(client, s, cid) + assert res["succeeded"] == [] and sorted(i for i, _ in res["failed"]) == ["a", "b"] + assert s.get_asset("a").processed is False # not marked processed on failure + assert s.already_applied("a", "trip", "Venice") is False # retryable + s.close() + + +def test_apply_non_trip(tmp_path): + s = _store(tmp_path) + cid = s.insert_cluster( + Cluster(start_at="2019-06-01", end_at="2019-06-02", status="non_trip"), + [ClusterMember(cluster_id=0, immich_id="a")]) + client = FakeImmich() + apply_cluster(client, s, cid) + assert ("id:_pipeline/non-trip", ["a"]) in client.tagged + assert ("id:_pipeline/processed", ["a"]) in client.tagged + s.close() + + +def test_apply_all_reports_per_cluster(tmp_path): + s = _store(tmp_path) + _approved(s, name="Venice") + client = FakeImmich() + results = apply_all(client, s) + assert len(results) == 1 and results[0]["status"] == "approved" + s.close() + + +def test_apply_skipped(tmp_path): + s = _store(tmp_path) + cid = s.insert_cluster( + Cluster(start_at="2019-06-01", end_at="2019-06-02", status="skipped"), + [ClusterMember(cluster_id=0, immich_id="a"), + ClusterMember(cluster_id=0, immich_id="b")]) + client = FakeImmich() + res = apply_cluster(client, s, cid) + # skipped: only _pipeline/processed is written, no content/non-trip tag + assert res["status"] == "skipped" + assert res["succeeded"] == [] and res["failed"] == [] + assert ("id:_pipeline/processed", ["a", "b"]) in client.tagged + assert len(client.tagged) == 1 # nothing but the processed tag + assert s.get_asset("a").processed is True + assert s.get_asset("b").processed is True + s.close() -- 2.54.0 From edf0735098171344dcaf7f26f8cd6b6a57b0d05e Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:35:58 +0200 Subject: [PATCH 15/19] feat(trip-cluster): master/detail review routes + templates + clusterReview JS --- apps/trip-cluster/app/routes/nav.py | 15 +- apps/trip-cluster/app/routes/review.py | 128 ++++++++++++- apps/trip-cluster/app/static/app.js | 182 +++++++++++++++++++ apps/trip-cluster/app/templates/_detail.html | 41 +++++ apps/trip-cluster/app/templates/review.html | 71 ++++++++ apps/trip-cluster/tests/test_routes.py | 74 ++++++++ 6 files changed, 508 insertions(+), 3 deletions(-) create mode 100644 apps/trip-cluster/app/static/app.js create mode 100644 apps/trip-cluster/app/templates/_detail.html create mode 100644 apps/trip-cluster/app/templates/review.html diff --git a/apps/trip-cluster/app/routes/nav.py b/apps/trip-cluster/app/routes/nav.py index 2245db8..fe0a50a 100644 --- a/apps/trip-cluster/app/routes/nav.py +++ b/apps/trip-cluster/app/routes/nav.py @@ -1,8 +1,14 @@ -from flask import Blueprint +from flask import Blueprint, current_app, render_template +from photoflow.core import Store bp = Blueprint("nav", __name__) +def _store(): + cfg = current_app.config["APP_CONFIG"] + return Store(cfg.db_path).connect() + + @bp.route("/health") def health(): return "ok" @@ -10,4 +16,9 @@ def health(): @bp.route("/") def index(): - return "trip-cluster" + s = _store() + clusters = s.clusters_by_attention() + stats = {"assets": len(s.all_assets()), "clusters": len(clusters), + "pending": sum(1 for c in clusters if c.status == "pending")} + s.close() + return render_template("review.html", clusters=clusters, stats=stats) diff --git a/apps/trip-cluster/app/routes/review.py b/apps/trip-cluster/app/routes/review.py index de13b75..0d26f18 100644 --- a/apps/trip-cluster/app/routes/review.py +++ b/apps/trip-cluster/app/routes/review.py @@ -1,3 +1,129 @@ -from flask import Blueprint +from flask import Blueprint, current_app, render_template, request, jsonify, abort +from photoflow.core import Store +from app import review bp = Blueprint("review", __name__) + + +def _store(): + cfg = current_app.config["APP_CONFIG"] + return Store(cfg.db_path).connect() + + +def _client(): + factory = current_app.config.get("IMMICH_FACTORY") + if factory: + return factory() + from photoflow.immich import ImmichClient + cfg = current_app.config["APP_CONFIG"] + return ImmichClient(cfg.immich_url, cfg.immich_api_key) + + +def _body(): + return request.get_json(silent=True) or {} + + +@bp.route("/cluster/") +def detail(cid): + s = _store() + c = s.get_cluster(cid) + if c is None: + s.close() + abort(404) + members = s.cluster_members(cid) + prev_id, next_id = s.chronological_neighbors(cid) + s.close() + return render_template("_detail.html", c=c, members=members, + prev_id=prev_id, next_id=next_id) + + +@bp.route("/cluster//approve", methods=["POST"]) +def approve(cid): + s = _store() + review.approve(s, cid, _body().get("name")) + c = s.get_cluster(cid) + s.close() + return jsonify({"ok": True, "status": c.status, "name": c.decided_name}) + + +@bp.route("/cluster//non-trip", methods=["POST"]) +def non_trip(cid): + s = _store() + review.mark_non_trip(s, cid) + s.close() + return jsonify({"ok": True, "status": "non_trip"}) + + +@bp.route("/cluster//skip", methods=["POST"]) +def skip(cid): + s = _store() + review.skip(s, cid) + s.close() + return jsonify({"ok": True, "status": "skipped"}) + + +@bp.route("/cluster//split", methods=["POST"]) +def split(cid): + s = _store() + boundary = _body().get("boundary_asset_id") + if not boundary: + s.close() + return jsonify({"error": "boundary_asset_id required"}), 400 + try: + id1, id2 = review.split(s, cid, boundary) + except ValueError as e: + s.close() + return jsonify({"error": str(e)}), 400 + s.close() + return jsonify({"ok": True, "ids": [id1, id2]}) + + +@bp.route("/cluster//merge", methods=["POST"]) +def merge(cid): + s = _store() + other = _body().get("other_id") + if other is None: + s.close() + return jsonify({"error": "other_id required"}), 400 + new_id = review.merge(s, cid, int(other)) + s.close() + return jsonify({"ok": True, "id": new_id}) + + +@bp.route("/cluster//member", methods=["POST"]) +def member(cid): + s = _store() + body = _body() + review.set_member(s, cid, body["asset_id"], bool(body.get("included"))) + s.close() + return jsonify({"ok": True}) + + +@bp.route("/approve-high-confidence", methods=["POST"]) +def approve_high_confidence(): + s = _store() + threshold = float(_body().get("threshold", 0.75)) + n = review.approve_high_confidence(s, threshold) + s.close() + return jsonify({"approved": n}) + + +@bp.route("/cluster//apply", methods=["POST"]) +def apply_one(cid): + # Lazy import: app.writeback is owned by a later task and may be absent + # at app-startup; importing here keeps the blueprint importable regardless. + from app.writeback import apply_cluster + s = _store() + res = apply_cluster(_client(), s, cid) + s.close() + return jsonify(res) + + +@bp.route("/apply-all", methods=["POST"]) +def apply_everything(): + # Lazy import: see apply_one above. + from app.writeback import apply_all + s = _store() + results = apply_all(_client(), s) + s.close() + return jsonify({"results": results}) diff --git a/apps/trip-cluster/app/static/app.js b/apps/trip-cluster/app/static/app.js new file mode 100644 index 0000000..7745dc0 --- /dev/null +++ b/apps/trip-cluster/app/static/app.js @@ -0,0 +1,182 @@ +function clusterReview() { + return { + ...photoGrid(), + selected: null, + clusterIds: [], + failures: [], + + init() { + this.clusterIds = [...document.querySelectorAll('.cluster-row')] + .map(el => parseInt(el.dataset.clusterId, 10)); + this.selected = this.clusterIds.length ? this.clusterIds[0] : null; + document.body.addEventListener('htmx:afterSwap', (e) => { + if (e.target.id === 'detail') { + this.hideIndicator(); + this.selectFirst(); + } + }); + }, + + // --- F16: loading indicator for programmatic swaps ----------------- + showIndicator() { + const el = document.getElementById('detail-indicator'); + if (el) el.classList.add('htmx-request'); + }, + hideIndicator() { + const el = document.getElementById('detail-indicator'); + if (el) el.classList.remove('htmx-request'); + }, + + selectCluster(id) { + this.selected = id; + this.showIndicator(); + htmx.ajax('GET', `/cluster/${id}`, { target: '#detail' }); + }, + + moveCluster(dir) { + const n = this.clusterIds.length; + if (!n) return; + const i = this.clusterIds.indexOf(this.selected); + const j = ((i + dir) % n + n) % n; // wrap around (mirrors nextPendingId) + if (this.clusterIds[j] != null) this.selectCluster(this.clusterIds[j]); + }, + + // --- F3: in-place rail status + advance to next pending ------------ + statusClass(status) { + return ({ + pending: 'badge-ghost', approved: 'badge-success', non_trip: 'badge-neutral', + skipped: 'badge-warning', merged: 'badge-info', split: 'badge-info', + })[status] || 'badge-ghost'; + }, + railStatusEl(id) { + return document.querySelector(`.cluster-status[data-cluster-id="${id}"]`); + }, + railStatusText(id) { + const el = this.railStatusEl(id); + return el ? el.textContent.trim() : ''; + }, + updateRailStatus(id, status) { + const el = this.railStatusEl(id); + if (el) el.innerHTML = + `${status}`; + }, + nextPendingId(fromId) { + const n = this.clusterIds.length; + const start = Math.max(0, this.clusterIds.indexOf(fromId)); + for (let k = 1; k <= n; k++) { + const id = this.clusterIds[(start + k) % n]; + if (id !== fromId && this.railStatusText(id) === 'pending') return id; + } + return null; + }, + // Update the acted cluster's rail badge in place (keeping the new status + // text visible) and advance the detail pane to the next pending cluster + // instead of bouncing back to the attention-queue head. + afterDecision(id, status) { + this.updateRailStatus(id, status); + const next = this.nextPendingId(id); + if (next != null) this.selectCluster(next); + else this.selectCluster(id); // no pending left: refresh current detail + }, + + nameValue() { + const el = document.getElementById('cluster-name'); + return el ? el.value : ''; + }, + + async post(path, body) { + const res = await fetch(path, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body || {}), + }); + return res.json(); + }, + + async approve() { + const acted = this.selected; + const r = await this.post(`/cluster/${acted}/approve`, { name: this.nameValue() }); + this.afterDecision(acted, r.status || 'approved'); + }, + async nonTrip() { + const acted = this.selected; + const r = await this.post(`/cluster/${acted}/non-trip`); + this.afterDecision(acted, r.status || 'non_trip'); + }, + async skip() { + const acted = this.selected; + const r = await this.post(`/cluster/${acted}/skip`); + this.afterDecision(acted, r.status || 'skipped'); + }, + async split() { + // F7: require a focused boundary photo before splitting. + if (!this.focused) { alert('Focus the first photo of the second trip, then Split.'); return; } + await this.post(`/cluster/${this.selected}/split`, { boundary_asset_id: this.focused.dataset.assetId }); + location.reload(); // split restructures the rail: full refresh is correct + }, + async merge(otherId) { + // F17: merge terminally marks both clusters merged with no UI un-merge. + if (!confirm('Merge these two clusters? This cannot be undone in the UI.')) return; + await this.post(`/cluster/${this.selected}/merge`, { other_id: otherId }); + location.reload(); // merge restructures the rail: full refresh is correct + }, + async setMember(assetId, included) { + await this.post(`/cluster/${this.selected}/member`, { asset_id: assetId, included }); + this.selectCluster(this.selected); + }, + async approveHighConfidence() { + // F19: show a pre-action count and confirm. + const threshold = 0.75; + const n = this.clusterIds.filter(id => { + const row = document.querySelector(`.cluster-row[data-cluster-id="${id}"]`); + return row && row.querySelector('.badge-success[title]'); // confidence "high" + }).length; + if (!confirm(`Approve ${n} high-confidence cluster(s) (>= ${threshold})?`)) return; + const r = await this.post('/approve-high-confidence', { threshold }); + alert(`Approved ${r.approved} cluster(s).`); + location.reload(); + }, + async apply() { + const r = await this.post(`/cluster/${this.selected}/apply`, {}); + alert(`Applied ${r.succeeded.length}, failed ${r.failed.length}.`); + }, + async applyAll() { + if (!confirm('Apply all approved decisions to Immich?')) return; + const r = await this.post('/apply-all', {}); + const results = r.results || []; + let ok = 0, bad = 0; + this.failures = []; + for (const x of results) { + const succeeded = x.succeeded || []; + const failed = x.failed || []; + ok += succeeded.length; + bad += failed.length; + if (failed.length) { + const id = x.cluster_id != null ? x.cluster_id : x.id; + const row = document.querySelector(`.cluster-row[data-cluster-id="${id}"]`); + if (row) row.classList.add('badge-error', 'ring-1', 'ring-error'); + this.failures.push({ + id, + name: row ? row.dataset.clusterName : `cluster ${id}`, + failed: failed.length, + }); + } + } + alert(`Applied ${ok} write(s), ${bad} failure(s).`); + }, + + onKey(e) { + if (e.target.tagName === 'INPUT') return; + const k = e.key.toLowerCase(); + if (e.key === '[') { e.preventDefault(); this.moveCluster(-1); } + else if (e.key === ']') { e.preventDefault(); this.moveCluster(1); } + else if (e.key === 'ArrowLeft') { e.preventDefault(); this.navigate(-1); } + else if (e.key === 'ArrowRight') { e.preventDefault(); this.navigate(1); } + else if (e.key === 'Enter') { if (this.focused) this.openLightbox(this.focused); } + else if (e.key === 'Escape') { this.closeLightbox(); } + else if (k === 'a') { this.approve(); } + else if (k === 'n') { this.nonTrip(); } + else if (k === 's') { if (this.focused) this.split(); } // F7: suppress until focused + else if (k === 'x') { this.skip(); } + }, + }; +} diff --git a/apps/trip-cluster/app/templates/_detail.html b/apps/trip-cluster/app/templates/_detail.html new file mode 100644 index 0000000..f371e95 --- /dev/null +++ b/apps/trip-cluster/app/templates/_detail.html @@ -0,0 +1,41 @@ +{% from "macros.html" import lightbox %} +
+
+ + + + + + + + {% if prev_id %}{% endif %} + {% if next_id %}{% endif %} +
+ +
+ Focus the first photo of the second trip (click it or use ← →), then Split. +
+
+ {{ c.start_at[:10] }} → {{ c.end_at[:10] }} · {{ members | length }} assets · status {{ c.status }} +
+
+ {% for a, m in members %} +
+ + {% if m.flagged_coverage %} + + {% elif m.is_outlier %} + + {% endif %} + {% if not m.included %}
{% endif %} +
+ {% endfor %} +
+ {{ lightbox() }} +
diff --git a/apps/trip-cluster/app/templates/review.html b/apps/trip-cluster/app/templates/review.html new file mode 100644 index 0000000..0a8a1ee --- /dev/null +++ b/apps/trip-cluster/app/templates/review.html @@ -0,0 +1,71 @@ +{% extends "base.html" %} +{% from "macros.html" import confidence_badge, status_badge %} +{% block title %}trip-cluster{% endblock %} +{% block navbar_title %}trip-cluster{% endblock %} +{% block content %} +{% if stats.clusters == 0 %} +
+

No clusters yet

+ {% if stats.assets == 0 %} +

Nothing ingested. Run + categorize ingest then categorize cluster.

+ {% else %} +

{{ stats.assets }} assets ingested, but this scope + produced no clusters. Try a wider categorize ingest scope.

+ {% endif %} +
+{% else %} +
+
+
+ Clusters + +
+ +

Needs attention first — lowest-confidence pending.

+ {% if stats.pending == 0 %} +
All reviewed — “Apply all” or categorize apply.
+ {% endif %} + +
+ Write failures: +
    + +
+
+ {% for c in clusters %} +
+
+ {{ c.decided_name or c.suggested_name }} + {{ confidence_badge(c.confidence) }} +
+
+ {{ status_badge(c.status) }} + {{ c.count }} + {{ c.start_at[:10] }} +
+
+ {% endfor %} + +
+
+ +
+ +
+
+
+
+
+ [ ] cluster · ← → grid · Enter open · A approve · N non-trip · S split (focus a photo) · X skip · Esc close +
+{% endif %} +{% endblock %} +{% block extra_scripts %}{% endblock %} diff --git a/apps/trip-cluster/tests/test_routes.py b/apps/trip-cluster/tests/test_routes.py index 18de583..e2a6036 100644 --- a/apps/trip-cluster/tests/test_routes.py +++ b/apps/trip-cluster/tests/test_routes.py @@ -1,6 +1,25 @@ import os from app import create_app from app.config import Config +from photoflow.core import Store +from photoflow.core.models import Asset, Cluster, ClusterMember + + +def _seed_store(cfg): + s = Store(cfg.db_path).connect() + for i, t in [("a", "2019-06-01"), ("b", "2019-06-02"), ("c", "2019-07-10")]: + s.upsert_asset(Asset(immich_id=i, taken_at=t)) + cid = s.insert_cluster( + Cluster(start_at="2019-06-01", end_at="2019-06-02", suggested_name="Venice", + confidence=0.9, kind_guess="trip", status="pending"), + [ClusterMember(cluster_id=0, immich_id="a"), + ClusterMember(cluster_id=0, immich_id="b")]) + other = s.insert_cluster( + Cluster(start_at="2019-07-10", end_at="2019-07-10", suggested_name="Rome", + confidence=0.3, kind_guess="trip", status="pending"), + [ClusterMember(cluster_id=0, immich_id="c")]) + s.close() + return cid, other def _app(tmp_path): @@ -24,3 +43,58 @@ def test_thumb_served(tmp_path): r = client.get("/thumb/a") assert r.status_code == 200 and r.mimetype == "image/jpeg" assert _app(tmp_path).test_client().get("/thumb/missing").status_code == 404 + + +def test_index_lists_clusters(tmp_path): + app = _app(tmp_path) + _seed_store(app.config["APP_CONFIG"]) + r = app.test_client().get("/") + assert r.status_code == 200 and b"Venice" in r.data and b"Rome" in r.data + + +def test_index_empty_state(tmp_path): + r = _app(tmp_path).test_client().get("/") + assert r.status_code == 200 and b"No clusters yet" in r.data + + +def test_detail_and_approve(tmp_path): + app = _app(tmp_path) + cid, _ = _seed_store(app.config["APP_CONFIG"]) + client = app.test_client() + d = client.get(f"/cluster/{cid}") + assert d.status_code == 200 and b"/thumb/a" in d.data + r = client.post(f"/cluster/{cid}/approve", json={"name": "Venezia"}) + assert r.get_json()["status"] == "approved" + s = Store(app.config["APP_CONFIG"].db_path).connect() + assert s.get_cluster(cid).decided_name == "Venezia" + s.close() + + +def test_member_toggle_and_high_confidence(tmp_path): + app = _app(tmp_path) + cid, _ = _seed_store(app.config["APP_CONFIG"]) + client = app.test_client() + client.post(f"/cluster/{cid}/member", json={"asset_id": "b", "included": False}) + s = Store(app.config["APP_CONFIG"].db_path).connect() + assert {m.immich_id: m.included for _, m in s.cluster_members(cid)}["b"] is False + s.close() + r = client.post("/approve-high-confidence", json={"threshold": 0.75}) + assert r.get_json()["approved"] == 1 # only the 0.9 cluster + + +def test_apply_all_with_injected_client(tmp_path): + app = _app(tmp_path) + cid, _ = _seed_store(app.config["APP_CONFIG"]) + + class FakeImmich: + def __init__(self): self.tagged = [] + def upsert_tag(self, name): return f"id:{name}" + def tag_assets(self, tid, ids): self.tagged.append((tid, list(ids))) + + fake = FakeImmich() + app.config["IMMICH_FACTORY"] = lambda: fake + client = app.test_client() + client.post(f"/cluster/{cid}/approve", json={"name": "Venice"}) + r = client.post("/apply-all", json={}) + assert r.status_code == 200 + assert any(tid == "id:Venice" for tid, _ in fake.tagged) -- 2.54.0 From c4ff5bcb06fb7001f42ed46f0570f7e66abb1cba Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:35:58 +0200 Subject: [PATCH 16/19] =?UTF-8?q?test(trip-cluster):=20Playwright=20UI=20?= =?UTF-8?q?=E2=80=94=20grid=20nav,=20lightbox,=20cluster=20switching,=20ap?= =?UTF-8?q?prove?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/trip-cluster/tests/ui/conftest.py | 60 ++++++++++++++++++++ apps/trip-cluster/tests/ui/test_review_ui.py | 45 +++++++++++++++ apps/trip-cluster/tests/ui/test_smoke_ui.py | 9 +++ 3 files changed, 114 insertions(+) create mode 100644 apps/trip-cluster/tests/ui/conftest.py create mode 100644 apps/trip-cluster/tests/ui/test_review_ui.py create mode 100644 apps/trip-cluster/tests/ui/test_smoke_ui.py diff --git a/apps/trip-cluster/tests/ui/conftest.py b/apps/trip-cluster/tests/ui/conftest.py new file mode 100644 index 0000000..0c0b675 --- /dev/null +++ b/apps/trip-cluster/tests/ui/conftest.py @@ -0,0 +1,60 @@ +import io +import os +import threading +import time + +import pytest +from PIL import Image +from werkzeug.serving import make_server + +from app import create_app +from app.config import Config +from photoflow.core import Store +from photoflow.core.models import Asset, Cluster, ClusterMember + + +def _jpeg(color): + buf = io.BytesIO() + Image.new("RGB", (48, 48), color).save(buf, format="JPEG") + return buf.getvalue() + + +@pytest.fixture(scope="session") +def data_dir(tmp_path_factory): + d = str(tmp_path_factory.mktemp("data")) + thumbs = os.path.join(d, "thumbs") + os.makedirs(thumbs, exist_ok=True) + s = Store(os.path.join(d, "trip-cluster.db")).connect() + palette = {"a": (10, 20, 30), "b": (40, 80, 120), "c": (200, 50, 90)} + for i, t in [("a", "2019-06-01"), ("b", "2019-06-02"), ("c", "2019-07-10")]: + s.upsert_asset(Asset(immich_id=i, taken_at=t)) + with open(os.path.join(thumbs, f"{i}.jpg"), "wb") as f: + f.write(_jpeg(palette[i])) + s.insert_cluster( + Cluster(start_at="2019-06-01", end_at="2019-06-02", suggested_name="Venice", + confidence=0.9, kind_guess="trip", status="pending"), + [ClusterMember(cluster_id=0, immich_id="a"), + ClusterMember(cluster_id=0, immich_id="b")]) + s.insert_cluster( + Cluster(start_at="2019-07-10", end_at="2019-07-10", suggested_name="Rome", + confidence=0.3, kind_guess="trip", status="pending"), + [ClusterMember(cluster_id=0, immich_id="c")]) + s.close() + return d + + +@pytest.fixture(scope="session") +def flask_app(data_dir): + cfg = Config(immich_url="http://127.0.0.1:1", immich_api_key="k", + anthropic_api_key="", data_dir=data_dir) + return create_app(cfg) + + +@pytest.fixture(scope="session") +def base_url(flask_app): + server = make_server("127.0.0.1", 8095, flask_app) + t = threading.Thread(target=server.serve_forever, daemon=True) + t.start() + time.sleep(0.2) + yield "http://127.0.0.1:8095" + server.shutdown() diff --git a/apps/trip-cluster/tests/ui/test_review_ui.py b/apps/trip-cluster/tests/ui/test_review_ui.py new file mode 100644 index 0000000..770c06d --- /dev/null +++ b/apps/trip-cluster/tests/ui/test_review_ui.py @@ -0,0 +1,45 @@ +def _open(page, base_url): + page.goto(base_url) + page.wait_for_selector(".photo-card") + + +def test_grid_arrow_navigation_moves_focus_ring(base_url, page): + # Venice (2 photos) is high-confidence; click it to load its detail. + _open(page, base_url) + page.click("text=Venice") + page.wait_for_selector(".photo-card") + cards = page.query_selector_all(".photo-card") + assert len(cards) == 2 + page.keyboard.press("ArrowRight") + # second card gains the focus ring class + assert "ring-4" in (page.query_selector_all(".photo-card")[1].get_attribute("class")) + + +def test_lightbox_opens_and_closes(base_url, page): + _open(page, base_url) + page.click("text=Venice") + page.wait_for_selector(".photo-card") + page.keyboard.press("ArrowRight") + page.keyboard.press("Enter") + assert page.is_visible("#lb") + page.keyboard.press("Escape") + page.wait_for_selector("#lb", state="hidden") + + +def test_cluster_switching_with_brackets(base_url, page): + _open(page, base_url) + page.click("text=Venice") + page.wait_for_selector(".photo-card") + page.keyboard.press("]") # move to the next cluster + page.wait_for_function( + "document.querySelectorAll('.photo-card').length === 1") # Rome has 1 photo + + +def test_approve_updates_status(base_url, page): + _open(page, base_url) + page.click("text=Venice") + page.wait_for_selector("#cluster-name") + page.once("dialog", lambda d: d.accept()) # no dialog expected, but be safe + page.click("text=Approve (A)") + page.wait_for_selector("text=approved") + assert "approved" in page.inner_text("body") diff --git a/apps/trip-cluster/tests/ui/test_smoke_ui.py b/apps/trip-cluster/tests/ui/test_smoke_ui.py new file mode 100644 index 0000000..7aeab7b --- /dev/null +++ b/apps/trip-cluster/tests/ui/test_smoke_ui.py @@ -0,0 +1,9 @@ +def test_health(base_url, page): + page.goto(f"{base_url}/health") + assert "ok" in page.content() + + +def test_index_lists_clusters(base_url, page): + page.goto(base_url) + assert "Venice" in page.inner_text("body") + assert "Rome" in page.inner_text("body") -- 2.54.0 From d67b677b6af5367a76af3d845b2a9a7d425d860a Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:37:22 +0200 Subject: [PATCH 17/19] docs+docker: README, CLAUDE.md, Dockerfile/compose (8084), M1 validation gate --- CLAUDE.md | 38 ++++++++++++++++++++++++++++++++ README.md | 37 +++++++++++++++++++++++++++++++ apps/trip-cluster/Dockerfile | 9 ++++++++ apps/trip-cluster/pyproject.toml | 3 +++ docker-compose.yml | 13 +++++++++++ docs/M1-validation-gate.md | 26 ++++++++++++++++++++++ 6 files changed, 126 insertions(+) create mode 100644 CLAUDE.md create mode 100644 README.md create mode 100644 apps/trip-cluster/Dockerfile create mode 100644 docker-compose.yml create mode 100644 docs/M1-validation-gate.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..84b0f66 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,38 @@ +# CLAUDE.md + +## What this is + +A monorepo (M1) of an Immich photo-flow foundation + the `trip-cluster` app. See +`docs/ROADMAP.md` for milestones and `docs/superpowers/specs/2026-06-27-immich-photo-flow-design.md` +for the M1 design. + +## Architecture map + +- `shared/photoflow/immich` — **the only** Immich client (`client.py`) + `_pipeline/` tag + conventions (`pipeline.py`). +- `shared/photoflow/core` — **the only** SQLite owner: `store.py` (data-access) + `models.py`. +- `shared/photoflow/ui` — `base.html`, shared Jinja macros, `shared.js` grid+lightbox. +- `apps/trip-cluster/app` — `config.py`, `cli.py` (ingest|cluster|serve|apply), + `ingest.py`, `clustering.py` (pure), `coverage.py` (pure), `cluster_run.py`, + `review.py`, `writeback.py`, `routes/`, `templates/`, `static/app.js`. + +## Key invariants + +- **Immich is the source of truth**; SQLite is rebuildable. Only **applied** decisions + survive loss of SQLite (via `_pipeline/processed` + `writeback_log`). +- Content/trip tags are **never namespaced**; pipeline meta-tags nest under `_pipeline/`. +- Trip detection is **timestamp-first, density-adaptive**, anchored by existing trip tags, + refined by GPS. Existing trip tags are authoritative seeds. +- Write-back is **idempotent** (`writeback_log`) and needs **explicit confirmation**. +- Ingest is **scopeable** (`--from/--to`, `--tag`, `--subset`) and incremental (`updatedAfter`). +- trip-cluster serves on **8084**. + +## Dev commands + +```bash +.venv/bin/python -m pytest # unit + route (Immich mocked) +.venv/bin/python -m pytest apps/trip-cluster/tests/ui # Playwright +docker compose up # UI on :8084 +``` + +Write a failing test first (TDD). Mirrors the sibling apps in `/home/mischa/Projects/`. diff --git a/README.md b/README.md new file mode 100644 index 0000000..381beaf --- /dev/null +++ b/README.md @@ -0,0 +1,37 @@ +# immich-photo-flow + +A monorepo of small tools that clean up and structure a large [Immich](https://immich.app) +library into travel "memories". **M1** ships the shared foundation +(`shared/photoflow/{immich,core,ui}`) and the `trip-cluster` app. + +## Layout + +- `shared/photoflow/immich` — the one Immich REST client + `_pipeline/` tag conventions. +- `shared/photoflow/core` — SQLite store + domain models (the only SQL). +- `shared/photoflow/ui` — base template, DaisyUI/Tailwind/Alpine/HTMX, shared grid+lightbox. +- `apps/trip-cluster` — CLI + Flask review UI (port 8084). + +## Dev setup + +```bash +python3.12 -m venv .venv +.venv/bin/pip install -e ./shared -e ./apps/trip-cluster \ + pytest==8.3.4 pytest-httpserver==1.1.0 pytest-playwright==0.6.2 +.venv/bin/python -m pytest # unit + route tests +.venv/bin/python -m playwright install chromium +.venv/bin/python -m pytest apps/trip-cluster/tests/ui # Playwright UI +``` + +## trip-cluster workflow + +```bash +cp .env.example .env # fill IMMICH_URL + IMMICH_API_KEY +cd apps/trip-cluster +python categorize.py ingest --tag "Italy 2019" # or --from/--to/--subset +python categorize.py cluster +python categorize.py serve # review at http://localhost:8084 +python categorize.py apply # write approved tags back (asks to confirm) +``` + +Immich is the source of truth; SQLite is a rebuildable working layer. Content/trip tags +are never namespaced; pipeline meta-tags live under `_pipeline/`. diff --git a/apps/trip-cluster/Dockerfile b/apps/trip-cluster/Dockerfile new file mode 100644 index 0000000..f1e26a5 --- /dev/null +++ b/apps/trip-cluster/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.12-slim +WORKDIR /srv +COPY shared/ ./shared/ +COPY apps/trip-cluster/ ./apps/trip-cluster/ +RUN pip install --no-cache-dir ./shared ./apps/trip-cluster +ENV DATA_DIR=/data +EXPOSE 8084 +WORKDIR /srv/apps/trip-cluster +CMD ["python", "categorize.py", "serve"] diff --git a/apps/trip-cluster/pyproject.toml b/apps/trip-cluster/pyproject.toml index f682724..645616d 100644 --- a/apps/trip-cluster/pyproject.toml +++ b/apps/trip-cluster/pyproject.toml @@ -11,3 +11,6 @@ dependencies = ["photoflow", "flask==3.1.0", "requests==2.32.3", "Pillow==11.0.0 [tool.setuptools.packages.find] where = ["."] include = ["app*"] + +[tool.setuptools.package-data] +"app" = ["templates/*.html", "static/*.js"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ad024c3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +services: + trip-cluster: + build: + context: . + dockerfile: apps/trip-cluster/Dockerfile + ports: + - "8084:8084" + volumes: + - ./data:/data + env_file: .env + environment: + - DATA_DIR=/data + user: "${UID}:${GID}" diff --git a/docs/M1-validation-gate.md b/docs/M1-validation-gate.md new file mode 100644 index 0000000..69e81d8 --- /dev/null +++ b/docs/M1-validation-gate.md @@ -0,0 +1,26 @@ +# M1 Validation Gate + +Before widening past the hard sample, trip-cluster must clear a quantified bar on a +**deliberately hard slice** — a GPS-poor, multi-year, low-density window (a well-remembered +old trip plus its surrounding everyday photos), **not** the easy phone-era last trip. + +## Protocol + +1. `categorize ingest --from --to ` (or `--tag `). +2. `categorize cluster`, then `categorize serve`. +3. Hand-label the slice: the true trip boundaries + which surrounding photos are non-trip. +4. Compare candidate clusters against the labels. + +## Acceptance bar (record actual numbers per run) + +| Metric | Definition | Target | +|--------|------------|--------| +| Trip-boundary precision | proposed boundaries that are real | ≥ 0.8 | +| Trip-boundary recall | real boundaries proposed | ≥ 0.8 | +| Coverage-flag recall | in-window missing-tag assets surfaced | ≥ 0.9 | +| Over-split rate | extra clusters per real trip | ≤ 0.5 | +| False-cluster rate | clusters that are pure noise | ≤ 0.1 | + +If unmet, tune `--gap-factor` / seed-span / confidence thresholds — do **not** widen the +backlog. See the F4 open question (everyday cluster blow-up): observe the real cluster +count on a representative subset first, then choose the surfacing/collapsing strategy. -- 2.54.0 From ae093881fa9b61c6470df96df420fb226fc877ae Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 17:54:38 +0200 Subject: [PATCH 18/19] fix(review): harden write-back, confirmation gate, and edge cases - write-back: call upsert_tag inside _apply_tag's try so a tag-create failure is caught per-asset and apply_all no longer aborts mid-batch (was P1) - write-back: surface _pipeline/processed write failures in the result instead of discarding them (was reported as success) - ui: add title to the high confidence badge so approve-high-confidence's pre-action count is non-zero; confirm() before single-cluster apply - core: Store context manager; close DB connection even if a route raises; guard split_cluster against a first-member boundary (empty cluster); add writeback_log lookup index - ingest: split thumbnail download/write error handling and clean up the .tmp file on a write failure - tests: upsert/processed write-failure regression tests + split-guard test --- apps/trip-cluster/app/ingest.py | 17 ++++++++--- apps/trip-cluster/app/routes/review.py | 10 +++---- apps/trip-cluster/app/static/app.js | 2 ++ apps/trip-cluster/app/writeback.py | 17 +++++++---- apps/trip-cluster/tests/test_writeback.py | 35 +++++++++++++++++++++++ shared/photoflow/core/store.py | 12 ++++++++ shared/photoflow/ui/templates/macros.html | 2 +- shared/tests/test_store_clusters.py | 11 +++++++ 8 files changed, 89 insertions(+), 17 deletions(-) diff --git a/apps/trip-cluster/app/ingest.py b/apps/trip-cluster/app/ingest.py index d207ad0..87e2ec6 100644 --- a/apps/trip-cluster/app/ingest.py +++ b/apps/trip-cluster/app/ingest.py @@ -45,14 +45,23 @@ def run_ingest(client, store, thumbs_dir, *, date_from=None, date_to=None, if need_thumb: try: data = client.download_thumbnail(a["id"]) - # Write to a temp path and os.replace into place so a failure - # never leaves a 0-byte .jpg behind. - tmp_path = f"{thumb_path}.tmp" + except Exception: + log.exception("Thumbnail download failed for asset %s", a["id"]) + continue + # Write to a temp path and os.replace into place so a failure + # never leaves a 0-byte .jpg behind; clean up the temp file + # if the write itself fails (e.g. disk full). + tmp_path = f"{thumb_path}.tmp" + try: with open(tmp_path, "wb") as f: f.write(data) os.replace(tmp_path, thumb_path) except Exception: - log.exception("Thumbnail download failed for asset %s", a["id"]) + log.exception("Thumbnail write failed for asset %s", a["id"]) + try: + os.unlink(tmp_path) + except OSError: + pass continue store.upsert_asset(Asset( diff --git a/apps/trip-cluster/app/routes/review.py b/apps/trip-cluster/app/routes/review.py index 0d26f18..6ee5805 100644 --- a/apps/trip-cluster/app/routes/review.py +++ b/apps/trip-cluster/app/routes/review.py @@ -113,9 +113,8 @@ def apply_one(cid): # Lazy import: app.writeback is owned by a later task and may be absent # at app-startup; importing here keeps the blueprint importable regardless. from app.writeback import apply_cluster - s = _store() - res = apply_cluster(_client(), s, cid) - s.close() + with _store() as s: # close even if the Immich write-back raises + res = apply_cluster(_client(), s, cid) return jsonify(res) @@ -123,7 +122,6 @@ def apply_one(cid): def apply_everything(): # Lazy import: see apply_one above. from app.writeback import apply_all - s = _store() - results = apply_all(_client(), s) - s.close() + with _store() as s: # close even if the Immich write-back raises + results = apply_all(_client(), s) return jsonify({"results": results}) diff --git a/apps/trip-cluster/app/static/app.js b/apps/trip-cluster/app/static/app.js index 7745dc0..9cd3235 100644 --- a/apps/trip-cluster/app/static/app.js +++ b/apps/trip-cluster/app/static/app.js @@ -136,6 +136,8 @@ function clusterReview() { location.reload(); }, async apply() { + // Write-back needs explicit confirmation (mirrors applyAll/merge). + if (!confirm('Apply this cluster’s decision to Immich?')) return; const r = await this.post(`/cluster/${this.selected}/apply`, {}); alert(`Applied ${r.succeeded.length}, failed ${r.failed.length}.`); }, diff --git a/apps/trip-cluster/app/writeback.py b/apps/trip-cluster/app/writeback.py index 7416583..663e94e 100644 --- a/apps/trip-cluster/app/writeback.py +++ b/apps/trip-cluster/app/writeback.py @@ -3,11 +3,15 @@ from photoflow.immich import pipeline APPLYABLE = ("approved", "non_trip", "skipped") -def _apply_tag(client, store, asset_ids, action, tag, tag_id): +def _apply_tag(client, store, asset_ids, action, tag): todo = [a for a in asset_ids if not store.already_applied(a, action, tag)] if not todo: return [], [] try: + # upsert_tag is part of the write: a failure here must be caught and + # recorded like a tag_assets failure, not propagate out of apply_cluster + # (which would abort the whole apply_all batch and skip later clusters). + tag_id = client.upsert_tag(tag) client.tag_assets(tag_id, todo) except Exception as e: # noqa: BLE001 — recorded, surfaced for a in todo: @@ -29,12 +33,11 @@ def apply_cluster(client, store, cluster_id) -> dict: if c.status == "approved": tag = c.decided_name or c.suggested_name - ok, fail = _apply_tag(client, store, included, "trip", tag, client.upsert_tag(tag)) + ok, fail = _apply_tag(client, store, included, "trip", tag) succeeded += ok failed += fail elif c.status == "non_trip": - ok, fail = _apply_tag(client, store, included, "non-trip", pipeline.NON_TRIP, - client.upsert_tag(pipeline.NON_TRIP)) + ok, fail = _apply_tag(client, store, included, "non-trip", pipeline.NON_TRIP) succeeded += ok failed += fail # 'skipped': no content/non-trip tag, only processed below. @@ -42,8 +45,10 @@ def apply_cluster(client, store, cluster_id) -> dict: failed_ids = {i for i, _ in failed} proc_targets = [a for a in included if a not in failed_ids] if proc_targets: - _apply_tag(client, store, proc_targets, "processed", pipeline.PROCESSED, - client.upsert_tag(pipeline.PROCESSED)) + # Surface processed-marker failures too — a discarded return here makes a + # failed _pipeline/processed write look like success in the CLI/UI summary. + _, proc_fail = _apply_tag(client, store, proc_targets, "processed", pipeline.PROCESSED) + failed += proc_fail for a in proc_targets: if store.already_applied(a, "processed", pipeline.PROCESSED): store.mark_processed(a) diff --git a/apps/trip-cluster/tests/test_writeback.py b/apps/trip-cluster/tests/test_writeback.py index 9deb8d6..7233c8e 100644 --- a/apps/trip-cluster/tests/test_writeback.py +++ b/apps/trip-cluster/tests/test_writeback.py @@ -8,9 +8,12 @@ class FakeImmich: def __init__(self): self.tagged = [] self.fail_tag_id = None + self.fail_upsert = None # tag name whose upsert_tag should raise self._ids = {} def upsert_tag(self, name): + if self.fail_upsert is not None and name == self.fail_upsert: + raise RuntimeError("upsert boom") self._ids.setdefault(name, f"id:{name}") return self._ids[name] @@ -111,3 +114,35 @@ def test_apply_skipped(tmp_path): assert s.get_asset("a").processed is True assert s.get_asset("b").processed is True s.close() + + +def test_upsert_tag_failure_is_caught_and_does_not_abort_batch(tmp_path): + # A failing upsert_tag must be recorded as a per-asset failure (retryable), + # not propagate out of apply_cluster and abort apply_all (review finding R1). + s = _store(tmp_path) + _approved(s, name="Venice") + _approved(s, name="Rome") # second cluster must still be applied + client = FakeImmich() + client.fail_upsert = "Venice" # first cluster's trip-tag upsert fails + results = apply_all(client, s) + assert len(results) == 2 # batch was not aborted by the first failure + venice = next(r for r in results if r["cluster_id"] == 1) + assert venice["succeeded"] == [] and sorted(i for i, _ in venice["failed"]) == ["a", "b"] + assert s.already_applied("a", "trip", "Venice") is False # retryable + rome = next(r for r in results if r["cluster_id"] == 2) + assert sorted(rome["succeeded"]) == ["a", "b"] + s.close() + + +def test_processed_write_failure_is_surfaced_in_result(tmp_path): + # Trip tag succeeds but the _pipeline/processed write fails: the failure must + # appear in result["failed"] and the asset must NOT be marked processed. + s = _store(tmp_path) + cid = _approved(s, name="Venice") + client = FakeImmich() + client.fail_upsert = pipeline.PROCESSED + res = apply_cluster(client, s, cid) + assert sorted(res["succeeded"]) == ["a", "b"] # trip tag still applied + assert sorted(i for i, _ in res["failed"]) == ["a", "b"] # processed surfaced + assert s.get_asset("a").processed is False + s.close() diff --git a/shared/photoflow/core/store.py b/shared/photoflow/core/store.py index 32c67ab..9fafa52 100644 --- a/shared/photoflow/core/store.py +++ b/shared/photoflow/core/store.py @@ -67,6 +67,8 @@ CREATE TABLE IF NOT EXISTS writeback_log ( CREATE INDEX IF NOT EXISTS idx_assets_taken_at ON assets(taken_at); CREATE INDEX IF NOT EXISTS idx_members_cluster ON cluster_members(cluster_id); CREATE INDEX IF NOT EXISTS idx_members_asset ON cluster_members(immich_id); +CREATE INDEX IF NOT EXISTS idx_writeback_lookup + ON writeback_log(immich_id, action, tag, result); """ @@ -109,6 +111,12 @@ class Store: self._conn.close() self._conn = None + def __enter__(self) -> "Store": + return self + + def __exit__(self, *exc) -> None: + self.close() + def upsert_asset(self, a: Asset) -> None: has_gps = 1 if (a.gps_lat is not None and a.gps_lon is not None) else 0 self.conn.execute( @@ -284,6 +292,10 @@ class Store: if boundary_immich_id not in ids: raise ValueError(f"boundary {boundary_immich_id!r} not in cluster") idx = ids.index(boundary_immich_id) + if idx == 0: + raise ValueError( + f"boundary {boundary_immich_id!r} is the first member; " + "split would leave an empty cluster") base = self.get_cluster(cluster_id) left, right = pairs[:idx], pairs[idx:] diff --git a/shared/photoflow/ui/templates/macros.html b/shared/photoflow/ui/templates/macros.html index 76430b0..872e6be 100644 --- a/shared/photoflow/ui/templates/macros.html +++ b/shared/photoflow/ui/templates/macros.html @@ -10,7 +10,7 @@ {% elif confidence < 0.75 %} med {% else %} - high + high {% endif %} {% endmacro %} diff --git a/shared/tests/test_store_clusters.py b/shared/tests/test_store_clusters.py index d8bddd8..bee53ad 100644 --- a/shared/tests/test_store_clusters.py +++ b/shared/tests/test_store_clusters.py @@ -97,3 +97,14 @@ def test_writeback_log_idempotency(tmp_path): s.log_writeback("b", "trip", "Venice", "error:boom") assert s.already_applied("b", "trip", "Venice") is False # only ok counts s.close() + + +def test_split_cluster_rejects_first_member_boundary(tmp_path): + import pytest + s = _seed(tmp_path) + cid = s.insert_cluster(Cluster(start_at="2019-06-01", end_at="2019-06-03", + suggested_name="Trip"), _members(["a", "b", "c"])) + with pytest.raises(ValueError): + s.split_cluster(cid, "a") # boundary == first member -> empty left + assert s.get_cluster(cid).status != "split" # original untouched on rejection + s.close() -- 2.54.0 From bd8fe59b49e162dc20d603901f3a857583233cb9 Mon Sep 17 00:00:00 2001 From: mischa Date: Sat, 27 Jun 2026 18:49:24 +0200 Subject: [PATCH 19/19] fix(ingest): don't advance incremental cursor past a failed/sampled asset A later successful asset could push last_ingest_at past an earlier asset whose thumbnail download failed, permanently excluding it from later incremental runs (only --full recovered it). Now the cursor never advances to/past the earliest failed asset, never below the prior cursor, and not at all on a --subset run. Regression tests added. (review finding #9) --- apps/trip-cluster/app/ingest.py | 23 ++++++++++++---- apps/trip-cluster/tests/test_ingest.py | 36 ++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/apps/trip-cluster/app/ingest.py b/apps/trip-cluster/app/ingest.py index 87e2ec6..c451b04 100644 --- a/apps/trip-cluster/app/ingest.py +++ b/apps/trip-cluster/app/ingest.py @@ -31,8 +31,9 @@ def run_ingest(client, store, thumbs_dir, *, date_from=None, date_to=None, assets = assets[:subset] processed_marked = 0 - max_updated = updated_after or "" tag_counts: dict = {} + successful_updated: list = [] # updated_at of assets ingested this run + failed_updated: list = [] # updated_at of assets we could NOT ingest for a in assets: # Revision (2): sanitize the Immich-provided id before using it as a @@ -47,6 +48,7 @@ def run_ingest(client, store, thumbs_dir, *, date_from=None, date_to=None, data = client.download_thumbnail(a["id"]) except Exception: log.exception("Thumbnail download failed for asset %s", a["id"]) + failed_updated.append(a["updated_at"]) continue # Write to a temp path and os.replace into place so a failure # never leaves a 0-byte .jpg behind; clean up the temp file @@ -62,6 +64,7 @@ def run_ingest(client, store, thumbs_dir, *, date_from=None, date_to=None, os.unlink(tmp_path) except OSError: pass + failed_updated.append(a["updated_at"]) continue store.upsert_asset(Asset( @@ -77,13 +80,23 @@ def run_ingest(client, store, thumbs_dir, *, date_from=None, date_to=None, store.mark_processed(a["id"]) processed_marked += 1 - if a["updated_at"] and a["updated_at"] > max_updated: - max_updated = a["updated_at"] + successful_updated.append(a["updated_at"]) for name, count in tag_counts.items(): store.upsert_tag(name, count=count) - if max_updated: - store.set_meta("last_ingest_at", max_updated) + # Advance the incremental cursor, but never past an asset we failed to ingest + # (a later success must not strand an earlier failure on the next run), never + # below the prior cursor, and not at all on a --subset sampling run. + if subset is None: + new_watermark = updated_after or "" + floor = min((u for u in failed_updated if u), default=None) + for u in successful_updated: + if not u or (floor is not None and u >= floor): + continue + if u > new_watermark: + new_watermark = u + if new_watermark: + store.set_meta("last_ingest_at", new_watermark) return {"fetched": len(assets), "processed_marked": processed_marked} diff --git a/apps/trip-cluster/tests/test_ingest.py b/apps/trip-cluster/tests/test_ingest.py index 25896b8..c0a7384 100644 --- a/apps/trip-cluster/tests/test_ingest.py +++ b/apps/trip-cluster/tests/test_ingest.py @@ -102,3 +102,39 @@ def test_ingest_resilient_to_thumb_failure_and_zero_byte(tmp_path): # The 0-byte file for "b" was replaced with real bytes. assert os.path.getsize(os.path.join(thumbs, "b.jpg")) > 0 store.close() + + +def test_ingest_watermark_not_advanced_past_failed_asset(tmp_path): + # A later successful asset must NOT advance last_ingest_at past an earlier + # asset whose thumbnail failed, or that asset is stranded on the next + # incremental run (review finding #9). + store = Store(str(tmp_path / "t.db")).connect() + thumbs = str(tmp_path / "thumbs"); os.makedirs(thumbs, exist_ok=True) + + class FailOld(FakeImmich): + def download_thumbnail(self, asset_id): + if asset_id == "old": + raise RuntimeError("boom") + return self._thumb + + client = FailOld([_asset("old", "2008-06-01", updated="2026-01-01T00:00:00Z"), + _asset("new", "2024-03-01", updated="2026-03-01T00:00:00Z")]) + run_ingest(client, store, thumbs) + assert store.get_asset("old") is None and store.get_asset("new") is not None + wm = store.get_meta("last_ingest_at") + # cursor must stay below the failed asset (here: not advanced at all) + assert wm is None or wm < "2026-01-01T00:00:00Z" + assert wm != "2026-03-01T00:00:00Z" + store.close() + + +def test_ingest_subset_run_does_not_advance_watermark(tmp_path): + # --subset is a sampling run; it must not move the incremental cursor or it + # would strand the un-fetched remainder. + store = Store(str(tmp_path / "t.db")).connect() + thumbs = str(tmp_path / "thumbs"); os.makedirs(thumbs, exist_ok=True) + client = FakeImmich([_asset("a", "2019-06-01", updated="2026-02-01T00:00:00Z"), + _asset("b", "2019-06-02", updated="2026-02-02T00:00:00Z")]) + run_ingest(client, store, thumbs, subset=1) + assert store.get_meta("last_ingest_at") is None + store.close() -- 2.54.0