fix(review): operator-agnostic sanity check + invariant TODOs + dedupe

Code-review follow-ups on the pgvector spike:
- self-similarity sanity check now compares to the seed's own self-distance
  (0 for cosine/L2, ~-1 for inner product <#>) instead of a hardcoded ~0, so it
  no longer misfires if Immich ever uses a vector_ip_ops index (correctness P3).
- mark the deliberate raw-SQLite read in the probe as a spike-only exception and
  add an M1.5 TODO that the real pgvector reader belongs in shared/photoflow/immich
  and SQLite access in shared/photoflow/core (project-standards P2 x2).
- document join_and_coverage's return shape; extract a _pct() helper to dedupe the
  coverage-percentage formatting (maintainability P3 x2).
Findings doc refreshed from the latest live run (coverage now ~46%, re-run ongoing).
This commit is contained in:
2026-06-27 20:50:06 +02:00
parent 7745f05323
commit 1a6afe86fc
3 changed files with 38 additions and 19 deletions
+30 -13
View File
@@ -176,20 +176,24 @@ def self_similarity_ok(conn, schema, table, column, key_col, op) -> str:
sql.SQL(
"WITH seed AS (SELECT {key} AS k, {col} AS v FROM {tbl} "
" WHERE {col} IS NOT NULL LIMIT 1) "
"SELECT (e.{key} = seed.k) AS is_seed, (e.{col} {op} seed.v) AS dist "
"SELECT (e.{key} = seed.k) AS is_seed, (e.{col} {op} seed.v) AS dist, "
" (seed.v {op} seed.v) AS self_dist "
"FROM {tbl} e, seed WHERE e.{col} IS NOT NULL ORDER BY dist ASC LIMIT 5"
).format(key=key, col=col, tbl=tbl, op=sql.SQL(op))
).fetchall()
if not rows:
return "skipped — no embeddings present"
is_seed, top_dist = rows[0]
# Operator works iff an identical vector reads back at distance ~0. The nearest
# row may be a duplicate photo (also dist 0) rather than the seed itself — still a pass.
good = abs(float(top_dist)) < 1e-6
is_seed, top_dist, self_dist = rows[0]
# Operator works iff an identical vector sorts first at the operator's own
# self-distance. That value is operator-dependent (0 for cosine <=> / L2 <->,
# ~-1 for inner product <#>), so compare the nearest distance to the seed's
# self-distance rather than to a hardcoded 0. The nearest row may be a duplicate
# photo rather than the seed itself — still a pass.
good = abs(float(top_dist) - float(self_dist)) < 1e-6
who = "the seed itself" if is_seed else "a duplicate (identical-vector) asset"
return (
f"{'OK' if good else 'UNEXPECTED'} — nearest neighbour ({who}) "
f"at distance {float(top_dist):.6g}"
f"at distance {float(top_dist):.6g} (seed self-distance {float(self_dist):.6g})"
)
except Exception as exc: # noqa: BLE001
return f"failed: {exc}"
@@ -244,7 +248,13 @@ def detect_asset_key(conn, schema, table) -> tuple[Optional[str], Optional[str],
def join_and_coverage(conn, schema, table, key_col, asset_table, assets_pk):
"""Join validity + coverage (raw and image-only) inside Postgres."""
"""Join validity + coverage (raw and image-only) inside Postgres.
Returns a dict (the contract its three callers read by key):
n_embeddings_distinct, n_joined_to_assets, n_orphan, n_assets_total: int
n_assets_image, n_embedded_image: int | None (None when no asset 'type' column)
raw_ratio, image_ratio: float | None (None when the denominator is 0/None)
"""
tbl, key = sql.Identifier(schema, table), sql.Identifier(key_col)
atbl, apk = sql.Identifier(schema, asset_table), sql.Identifier(assets_pk)
@@ -300,6 +310,10 @@ def sqlite_crosscheck(conn, schema, table, key_col, sqlite_path: Path) -> str:
Degrades gracefully: a fresh checkout has no `ingest` run, so an empty/absent
store is not a failure of the join requirement.
NOTE: this opens SQLite directly, a deliberate exception for this throwaway probe.
Production M1.5 code must NOT — it must go through shared/photoflow/core/store.py,
the only SQLite owner (CLAUDE.md). This raw read-only sample check stays in the spike.
"""
if not sqlite_path.is_file():
return f"skipped — no SQLite store at {sqlite_path} (fresh checkout; not a failure)"
@@ -365,10 +379,15 @@ def environment_meta(conn) -> dict:
# -------------------------------------------------------------------------- output
def _pct(ratio: Optional[float]) -> str:
"""Format a coverage ratio as a percentage, or 'n/a' when it could not be computed."""
return "n/a" if ratio is None else f"{ratio * 100:.1f}%"
def render_findings(facts: dict) -> str:
cov = facts["coverage"]
raw_pct = "n/a" if cov["raw_ratio"] is None else f"{cov['raw_ratio'] * 100:.1f}%"
img_pct = "n/a" if cov["image_ratio"] is None else f"{cov['image_ratio'] * 100:.1f}%"
raw_pct = _pct(cov["raw_ratio"])
img_pct = _pct(cov["image_ratio"])
meta = facts["meta"]
return textwrap.dedent(
f"""\
@@ -469,10 +488,8 @@ def print_report(facts: dict) -> None:
print(f" operator sanity : {facts['self_similarity']}")
print(f" asset key : {facts['asset_key_note']}")
print(f" join : {facts['join_summary']} (orphans: {cov['n_orphan']})")
raw = "n/a" if cov["raw_ratio"] is None else f"{cov['raw_ratio'] * 100:.1f}%"
img = "n/a" if cov["image_ratio"] is None else f"{cov['image_ratio'] * 100:.1f}%"
print(f" coverage (raw) : {raw} ({cov['n_joined_to_assets']}/{cov['n_assets_total']})")
print(f" coverage (image) : {img} ({cov['n_embedded_image']}/{cov['n_assets_image']})")
print(f" coverage (raw) : {_pct(cov['raw_ratio'])} ({cov['n_joined_to_assets']}/{cov['n_assets_total']})")
print(f" coverage (image) : {_pct(cov['image_ratio'])} ({cov['n_embedded_image']}/{cov['n_assets_image']})")
print(f" sqlite x-check : {facts['sqlite']}")
print(f" postgres : {facts['meta']['postgres_version']}")
print(f" pgvector ext : {facts['meta']['pgvector_extension']}")