338 lines
14 KiB
Python
338 lines
14 KiB
Python
import sqlite3
|
|
|
|
from photoflow.core.models import Asset, Cluster, ClusterMember, Tag
|
|
|
|
SCHEMA_VERSION = 1
|
|
|
|
SCHEMA = """
|
|
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT);
|
|
|
|
CREATE TABLE IF NOT EXISTS assets (
|
|
immich_id TEXT PRIMARY KEY,
|
|
taken_at TEXT,
|
|
gps_lat REAL, gps_lon REAL,
|
|
place_city TEXT, place_country TEXT,
|
|
type TEXT,
|
|
has_gps INTEGER NOT NULL DEFAULT 0,
|
|
thumb_path TEXT,
|
|
processed INTEGER NOT NULL DEFAULT 0,
|
|
ingested_at TEXT,
|
|
updated_at TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS tags (
|
|
name TEXT PRIMARY KEY,
|
|
immich_tag_id TEXT,
|
|
count INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS asset_tags (
|
|
immich_id TEXT NOT NULL,
|
|
tag_name TEXT NOT NULL,
|
|
PRIMARY KEY (immich_id, tag_name)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS clusters (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
start_at TEXT, end_at TEXT,
|
|
count INTEGER NOT NULL DEFAULT 0,
|
|
suggested_name TEXT,
|
|
confidence REAL NOT NULL DEFAULT 0,
|
|
kind_guess TEXT NOT NULL DEFAULT 'trip',
|
|
status TEXT NOT NULL DEFAULT 'pending',
|
|
decided_name TEXT,
|
|
reviewed_at TEXT,
|
|
notes TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS cluster_members (
|
|
cluster_id INTEGER NOT NULL,
|
|
immich_id TEXT NOT NULL,
|
|
member_confidence REAL NOT NULL DEFAULT 1.0,
|
|
is_outlier INTEGER NOT NULL DEFAULT 0,
|
|
included INTEGER NOT NULL DEFAULT 1,
|
|
flagged_coverage INTEGER NOT NULL DEFAULT 0,
|
|
PRIMARY KEY (cluster_id, immich_id)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS writeback_log (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
immich_id TEXT NOT NULL,
|
|
action TEXT NOT NULL,
|
|
tag TEXT,
|
|
result TEXT NOT NULL,
|
|
applied_at TEXT NOT NULL
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_assets_taken_at ON assets(taken_at);
|
|
CREATE INDEX IF NOT EXISTS idx_members_cluster ON cluster_members(cluster_id);
|
|
CREATE INDEX IF NOT EXISTS idx_members_asset ON cluster_members(immich_id);
|
|
"""
|
|
|
|
|
|
class Store:
|
|
def __init__(self, db_path: str):
|
|
self.db_path = db_path
|
|
self._conn = None
|
|
|
|
def connect(self) -> "Store":
|
|
self._conn = sqlite3.connect(self.db_path)
|
|
self._conn.row_factory = sqlite3.Row
|
|
self._conn.execute("PRAGMA foreign_keys=ON")
|
|
self.migrate()
|
|
return self
|
|
|
|
@property
|
|
def conn(self) -> sqlite3.Connection:
|
|
if self._conn is None:
|
|
raise RuntimeError("Store not connected; call connect()")
|
|
return self._conn
|
|
|
|
def migrate(self) -> None:
|
|
self.conn.executescript(SCHEMA)
|
|
if self.get_meta("schema_version") is None:
|
|
self.set_meta("schema_version", str(SCHEMA_VERSION))
|
|
self.conn.commit()
|
|
|
|
def set_meta(self, key: str, value: str) -> None:
|
|
self.conn.execute(
|
|
"INSERT INTO meta(key, value) VALUES(?, ?) "
|
|
"ON CONFLICT(key) DO UPDATE SET value=excluded.value", (key, value))
|
|
self.conn.commit()
|
|
|
|
def get_meta(self, key: str, default=None):
|
|
row = self.conn.execute("SELECT value FROM meta WHERE key=?", (key,)).fetchone()
|
|
return row["value"] if row else default
|
|
|
|
def close(self) -> None:
|
|
if self._conn is not None:
|
|
self._conn.close()
|
|
self._conn = None
|
|
|
|
def upsert_asset(self, a: Asset) -> None:
|
|
has_gps = 1 if (a.gps_lat is not None and a.gps_lon is not None) else 0
|
|
self.conn.execute(
|
|
"""INSERT INTO assets
|
|
(immich_id, taken_at, gps_lat, gps_lon, place_city, place_country,
|
|
type, has_gps, thumb_path, processed, ingested_at, updated_at)
|
|
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
|
|
ON CONFLICT(immich_id) DO UPDATE SET
|
|
taken_at=excluded.taken_at, gps_lat=excluded.gps_lat,
|
|
gps_lon=excluded.gps_lon, place_city=excluded.place_city,
|
|
place_country=excluded.place_country, type=excluded.type,
|
|
has_gps=excluded.has_gps, thumb_path=excluded.thumb_path,
|
|
ingested_at=excluded.ingested_at, updated_at=excluded.updated_at""",
|
|
(a.immich_id, a.taken_at, a.gps_lat, a.gps_lon, a.place_city,
|
|
a.place_country, a.type, has_gps, a.thumb_path,
|
|
1 if a.processed else 0, a.ingested_at, a.updated_at))
|
|
self.conn.commit()
|
|
|
|
def _asset_from_row(self, r) -> Asset:
|
|
return Asset(
|
|
immich_id=r["immich_id"], taken_at=r["taken_at"],
|
|
gps_lat=r["gps_lat"], gps_lon=r["gps_lon"],
|
|
place_city=r["place_city"], place_country=r["place_country"],
|
|
type=r["type"], has_gps=bool(r["has_gps"]), thumb_path=r["thumb_path"],
|
|
processed=bool(r["processed"]), ingested_at=r["ingested_at"],
|
|
updated_at=r["updated_at"])
|
|
|
|
def get_asset(self, immich_id: str):
|
|
r = self.conn.execute("SELECT * FROM assets WHERE immich_id=?", (immich_id,)).fetchone()
|
|
return self._asset_from_row(r) if r else None
|
|
|
|
def all_assets(self, include_processed: bool = True) -> list:
|
|
sql = "SELECT * FROM assets"
|
|
if not include_processed:
|
|
sql += " WHERE processed=0"
|
|
sql += " ORDER BY taken_at"
|
|
return [self._asset_from_row(r) for r in self.conn.execute(sql)]
|
|
|
|
def assets_in_range(self, start_at: str, end_at: str) -> list:
|
|
return [self._asset_from_row(r) for r in self.conn.execute(
|
|
"SELECT * FROM assets WHERE taken_at>=? AND taken_at<=? ORDER BY taken_at",
|
|
(start_at, end_at))]
|
|
|
|
def mark_processed(self, immich_id: str) -> None:
|
|
self.conn.execute("UPDATE assets SET processed=1 WHERE immich_id=?", (immich_id,))
|
|
self.conn.commit()
|
|
|
|
def set_asset_tags(self, immich_id: str, tag_names: list) -> None:
|
|
self.conn.execute("DELETE FROM asset_tags WHERE immich_id=?", (immich_id,))
|
|
self.conn.executemany(
|
|
"INSERT OR IGNORE INTO asset_tags(immich_id, tag_name) VALUES(?, ?)",
|
|
[(immich_id, n) for n in tag_names])
|
|
self.conn.commit()
|
|
|
|
def asset_tags(self, immich_id: str) -> list:
|
|
return [r["tag_name"] for r in self.conn.execute(
|
|
"SELECT tag_name FROM asset_tags WHERE immich_id=? ORDER BY tag_name",
|
|
(immich_id,))]
|
|
|
|
def upsert_tag(self, name: str, immich_tag_id=None, count: int = 0) -> None:
|
|
self.conn.execute(
|
|
"""INSERT INTO tags(name, immich_tag_id, count) VALUES(?,?,?)
|
|
ON CONFLICT(name) DO UPDATE SET
|
|
immich_tag_id=excluded.immich_tag_id, count=excluded.count""",
|
|
(name, immich_tag_id, count))
|
|
self.conn.commit()
|
|
|
|
def all_tags(self) -> list:
|
|
return [Tag(name=r["name"], immich_tag_id=r["immich_tag_id"], count=r["count"])
|
|
for r in self.conn.execute("SELECT * FROM tags ORDER BY name")]
|
|
|
|
def insert_cluster(self, c: Cluster, members: list) -> int:
|
|
cur = self.conn.execute(
|
|
"""INSERT INTO clusters
|
|
(start_at, end_at, count, suggested_name, confidence, kind_guess,
|
|
status, decided_name, reviewed_at, notes)
|
|
VALUES (?,?,?,?,?,?,?,?,?,?)""",
|
|
(c.start_at, c.end_at, c.count or len(members), c.suggested_name,
|
|
c.confidence, c.kind_guess, c.status, c.decided_name, c.reviewed_at, c.notes))
|
|
cid = cur.lastrowid
|
|
self.conn.executemany(
|
|
"""INSERT OR REPLACE INTO cluster_members
|
|
(cluster_id, immich_id, member_confidence, is_outlier, included, flagged_coverage)
|
|
VALUES (?,?,?,?,?,?)""",
|
|
[(cid, m.immich_id, m.member_confidence, 1 if m.is_outlier else 0,
|
|
1 if m.included else 0, 1 if m.flagged_coverage else 0) for m in members])
|
|
self.conn.commit()
|
|
return cid
|
|
|
|
def clear_clusters(self) -> None:
|
|
self.conn.execute("DELETE FROM cluster_members")
|
|
self.conn.execute("DELETE FROM clusters")
|
|
self.conn.commit()
|
|
|
|
def _cluster_from_row(self, r) -> Cluster:
|
|
return Cluster(
|
|
id=r["id"], start_at=r["start_at"], end_at=r["end_at"], count=r["count"],
|
|
suggested_name=r["suggested_name"], confidence=r["confidence"],
|
|
kind_guess=r["kind_guess"], status=r["status"], decided_name=r["decided_name"],
|
|
reviewed_at=r["reviewed_at"], notes=r["notes"])
|
|
|
|
def get_cluster(self, cluster_id: int):
|
|
r = self.conn.execute("SELECT * FROM clusters WHERE id=?", (cluster_id,)).fetchone()
|
|
return self._cluster_from_row(r) if r else None
|
|
|
|
def all_clusters(self) -> list:
|
|
return [self._cluster_from_row(r)
|
|
for r in self.conn.execute("SELECT * FROM clusters ORDER BY start_at, id")]
|
|
|
|
def clusters_by_attention(self) -> list:
|
|
return [self._cluster_from_row(r) for r in self.conn.execute(
|
|
"""SELECT * FROM clusters
|
|
ORDER BY CASE status WHEN 'pending' THEN 0 ELSE 1 END,
|
|
confidence ASC,
|
|
(julianday(end_at) - julianday(start_at)) DESC,
|
|
id""")]
|
|
|
|
def cluster_members(self, cluster_id: int) -> list:
|
|
rows = self.conn.execute(
|
|
"""SELECT a.*, m.member_confidence AS m_conf, m.is_outlier AS m_out,
|
|
m.included AS m_inc, m.flagged_coverage AS m_cov
|
|
FROM cluster_members m JOIN assets a ON a.immich_id = m.immich_id
|
|
WHERE m.cluster_id=? ORDER BY a.taken_at, a.immich_id""", (cluster_id,))
|
|
out = []
|
|
for r in rows:
|
|
asset = self._asset_from_row(r)
|
|
member = ClusterMember(
|
|
cluster_id=cluster_id, immich_id=r["immich_id"],
|
|
member_confidence=r["m_conf"], is_outlier=bool(r["m_out"]),
|
|
included=bool(r["m_inc"]), flagged_coverage=bool(r["m_cov"]))
|
|
out.append((asset, member))
|
|
return out
|
|
|
|
def chronological_neighbors(self, cluster_id: int):
|
|
ids = [r["id"] for r in self.conn.execute(
|
|
"SELECT id FROM clusters ORDER BY start_at, id")]
|
|
if cluster_id not in ids:
|
|
return (None, None)
|
|
i = ids.index(cluster_id)
|
|
prev_id = ids[i - 1] if i > 0 else None
|
|
next_id = ids[i + 1] if i < len(ids) - 1 else None
|
|
return (prev_id, next_id)
|
|
|
|
def update_cluster(self, cluster_id: int, *, status=None, decided_name=None,
|
|
suggested_name=None, notes=None, reviewed_at=None) -> None:
|
|
sets, vals = [], []
|
|
for col, val in [("status", status), ("decided_name", decided_name),
|
|
("suggested_name", suggested_name), ("notes", notes),
|
|
("reviewed_at", reviewed_at)]:
|
|
if val is not None:
|
|
sets.append(f"{col}=?")
|
|
vals.append(val)
|
|
if not sets:
|
|
return
|
|
vals.append(cluster_id)
|
|
self.conn.execute(f"UPDATE clusters SET {', '.join(sets)} WHERE id=?", vals)
|
|
self.conn.commit()
|
|
|
|
def set_member_inclusion(self, cluster_id: int, immich_id: str, included: bool) -> None:
|
|
self.conn.execute(
|
|
"UPDATE cluster_members SET included=? WHERE cluster_id=? AND immich_id=?",
|
|
(1 if included else 0, cluster_id, immich_id))
|
|
self.conn.commit()
|
|
|
|
def _recompute_span(self, members: list) -> tuple:
|
|
pairs = members
|
|
starts = [a.taken_at for a, _ in pairs]
|
|
return (min(starts), max(starts)) if starts else ("", "")
|
|
|
|
def split_cluster(self, cluster_id: int, boundary_immich_id: str):
|
|
pairs = self.cluster_members(cluster_id)
|
|
ids = [a.immich_id for a, _ in pairs]
|
|
if boundary_immich_id not in ids:
|
|
raise ValueError(f"boundary {boundary_immich_id!r} not in cluster")
|
|
idx = ids.index(boundary_immich_id)
|
|
base = self.get_cluster(cluster_id)
|
|
left, right = pairs[:idx], pairs[idx:]
|
|
|
|
def _new(part, suffix):
|
|
ms = [ClusterMember(cluster_id=0, immich_id=a.immich_id,
|
|
member_confidence=m.member_confidence,
|
|
is_outlier=m.is_outlier, included=m.included,
|
|
flagged_coverage=m.flagged_coverage) for a, m in part]
|
|
start, end = self._recompute_span(part)
|
|
return self.insert_cluster(Cluster(
|
|
start_at=start, end_at=end, count=len(ms),
|
|
suggested_name=f"{base.suggested_name} ({suffix})",
|
|
confidence=base.confidence, kind_guess=base.kind_guess,
|
|
status="pending"), ms)
|
|
|
|
id1, id2 = _new(left, 1), _new(right, 2)
|
|
self.update_cluster(cluster_id, status="split")
|
|
return (id1, id2)
|
|
|
|
def merge_clusters(self, cluster_id_a: int, cluster_id_b: int):
|
|
pairs = self.cluster_members(cluster_id_a) + self.cluster_members(cluster_id_b)
|
|
pairs.sort(key=lambda p: (p[0].taken_at, p[0].immich_id))
|
|
base = self.get_cluster(cluster_id_a)
|
|
ms = [ClusterMember(cluster_id=0, immich_id=a.immich_id,
|
|
member_confidence=m.member_confidence, is_outlier=m.is_outlier,
|
|
included=m.included, flagged_coverage=m.flagged_coverage)
|
|
for a, m in pairs]
|
|
start, end = self._recompute_span(pairs)
|
|
new = self.insert_cluster(Cluster(
|
|
start_at=start, end_at=end, count=len(ms),
|
|
suggested_name=base.suggested_name, confidence=base.confidence,
|
|
kind_guess=base.kind_guess, status="pending"), ms)
|
|
self.update_cluster(cluster_id_a, status="merged")
|
|
self.update_cluster(cluster_id_b, status="merged")
|
|
return new
|
|
|
|
def log_writeback(self, immich_id: str, action: str, tag, result: str) -> None:
|
|
import datetime
|
|
self.conn.execute(
|
|
"INSERT INTO writeback_log(immich_id, action, tag, result, applied_at) "
|
|
"VALUES (?,?,?,?,?)",
|
|
(immich_id, action, tag, result,
|
|
datetime.datetime.now(datetime.timezone.utc).isoformat()))
|
|
self.conn.commit()
|
|
|
|
def already_applied(self, immich_id: str, action: str, tag) -> bool:
|
|
row = self.conn.execute(
|
|
"SELECT 1 FROM writeback_log WHERE immich_id=? AND action=? "
|
|
"AND IFNULL(tag,'')=IFNULL(?, '') AND result='ok' LIMIT 1",
|
|
(immich_id, action, tag)).fetchone()
|
|
return row is not None
|