Merge branch 'main' into worktree-content-fixes

Makefile

@@ -27,6 +27,7 @@ REMOTE_TARGETS := remote-env-setup remote-env-remove remote-wipe remote-install
 	remote-fetch remote-fetch-content remote-install-plugins remote-update-plugins \
 	remote-upgrade-grav remote-git-sync-disable remote-git-sync-enable \
 	remote-content-status remote-clean remote-diag remote-apply-env \
+	remote-seed-api-salt remote-secrets-audit \
 	remote-gpm-install remote-maintenance-on remote-maintenance-off
 ENVS := test prod
 
@@ -178,7 +179,7 @@ remote-git-sync-enable: guard-env
 	$(SSH) "bash -s -- '$(WEBROOT)' true" < scripts/git-sync-toggle.sh
 
 remote-content-status: guard-env
-	$(SSH) "cd $(WEBROOT)/user && git status --short && echo '--- config diff ---' && git diff -- config/"
+	$(SSH) "cd $(WEBROOT)/user && echo '--- HEAD ---' && git log -1 --oneline && echo '--- working tree ---' && git status --short && echo '--- config diff ---' && git diff -- config/ && echo '--- .gitignore diff ---' && git diff -- .gitignore"
 
 remote-clean: guard-env
 	$(SSH) "cd $(WEBROOT) && php bin/grav clearcache"
@@ -202,6 +203,31 @@ remote-apply-env: guard-env
 	  $(SSH) "mkdir -p $(WEBROOT)/user/env/$$host/config && cat > $(WEBROOT)/user/env/$$host/config/system.yaml && cd $(WEBROOT) && php bin/grav clearcache" < deploy/env/$(ENV)/system.yaml; \
 	  echo "Applied deploy/env/$(ENV)/system.yaml -> $(WEBROOT)/user/env/$$host/config/system.yaml"
 
+# Seed a per-host popularity salt into the env override tree so the api plugin
+# reads it there instead of appending one to the git-tracked config/plugins/
+# api.yaml. That appended salt kept the content working tree dirty, which broke
+# git-sync's auto-merge on webhook. Salt is generated server-side and never
+# committed (a committed salt would be globally known). Idempotent: an existing
+# salt is kept, so re-running never rotates it.
+remote-seed-api-salt: guard-env
+	@host="$${WEB_HOST:-$(REMOTE_HOST)}"; \
+	  test -n "$$host" || { echo "ERROR: WEB_HOST/REMOTE_HOST unresolved"; exit 1; }; \
+	  $(SSH) "set -e; \
+	    envfile=$(WEBROOT)/user/env/$$host/config/plugins/api.yaml; \
+	    mkdir -p \$$(dirname \"\$$envfile\"); \
+	    if grep -qE '^[[:space:]]*salt:' \"\$$envfile\" 2>/dev/null; then \
+	      echo \"salt already present in \$$envfile — keeping it\"; \
+	    else \
+	      salt=\$$(openssl rand -hex 32); \
+	      printf 'popularity:\n  salt: %s\n' \"\$$salt\" > \"\$$envfile\"; \
+	      echo \"seeded new per-host salt into \$$envfile\"; \
+	    fi; \
+	    git -C $(WEBROOT)/user checkout -- config/plugins/api.yaml 2>/dev/null || true; \
+	    cd $(WEBROOT) && php bin/grav clearcache >/dev/null 2>&1 || true; \
+	    echo '--- base api.yaml status (expect clean) ---'; \
+	    git -C $(WEBROOT)/user status --short config/plugins/api.yaml; \
+	    echo '(if the line above is empty, the tree is clean)'"
+
 # Read-only health check: plugin install state, versions, key config, log tail.
 remote-diag: guard-env
 	$(SSH) "cd $(WEBROOT) && \
@@ -214,6 +240,16 @@ remote-diag: guard-env
 		echo '=== git-sync config (secrets redacted) ==='; grep -vaiE 'password|token|secret' user/config/plugins/git-sync.yaml user/env/*/config/plugins/git-sync.yaml 2>/dev/null; \
 		echo '=== grav.log tail ==='; tail -8 logs/grav.log 2>/dev/null"
 
+# Secret-safe audit: lists WHERE per-host secret/config files live (config/ vs
+# env/<host>/config/) and their sizes — never prints contents. Used to decide
+# whether a `reset --hard` would clobber a live runtime secret.
+remote-secrets-audit: guard-env
+	$(SSH) "cd $(WEBROOT)/user && \
+		echo '=== tracked in git? (git ls-files) ==='; git ls-files config/security-private.php config/security.yaml config/versions.yaml config/plugins/api-private.php config/plugins/git-sync.yaml; \
+		echo '=== config/ copies (size only) ==='; ls -la config/security.yaml config/security-private.php config/versions.yaml config/plugins/api-private.php config/plugins/git-sync.yaml 2>&1; \
+		echo '=== env/<host>/config copies (size only) ==='; ls -la env/*/config/security.yaml env/*/config/security-private.php env/*/config/plugins/api-private.php env/*/config/plugins/git-sync.yaml 2>&1; \
+		echo '=== does security.yaml reference the private php? (key names only) ==='; grep -aoE '^[a-z_]+:' config/security.yaml 2>/dev/null; for f in env/*/config/security.yaml; do echo \"\$$f:\"; grep -aoE '^[a-z_]+:' \"\$$f\" 2>/dev/null; done; true"
+
 remote-maintenance-on: guard-env
 	$(SSH) "bash -s on $(WEBROOT)" < scripts/server-maintenance.sh
 

docs/solutions/architecture-patterns/git-sync-secret-exposure-and-tracked-file-boomerang.md

@@ -1,6 +1,7 @@
 ---
 title: "Secret exposure under bidirectional Grav git-sync: gitignore is the only boundary (and the tracked-file boomerang trap)"
 date: 2026-07-05
+last_updated: 2026-07-05
 module: git-sync
 problem_type: architecture_pattern
 component: tooling
@@ -174,6 +175,69 @@ committing looks clean locally, but Grav re-appends it at runtime and the next
 sync re-commits and re-pushes it. The only durable fixes are the
 companion-private-file pattern or disabling the feature.
 
+### Untracking an already-committed secret under a live sync — freeze every server first
+
+`.gitignore` (and git-sync's `ignore` field) only affects **untracked** files.
+Once a secret has actually been *committed* to the shared repo, ignoring it does
+nothing — removing it means rewriting history. Doing that while a bidirectional
+sync is live has its own trap.
+
+**A `direction: both` server silently reverts your force-push.** Concrete
+incident (2026-07-05, a second occurrence of this doc's trap): `config/security-private.php`
+and `config/versions.yaml` had been committed to Gitea `main` (auto-commit
+`d1643a7`, merged at `f8c45fc`). Force-pushing `main` back to the clean commit
+`32c3d8c` *looked* successful — and within seconds Gitea was back at `f8c45fc`.
+Cause: prod's git-sync is `direction: both` and its local `HEAD` was still
+`f8c45fc`; on its next sync it re-pushed the stale, secret-bearing commit and
+undid the rewrite. The pull-only test server never fought back — **only
+push-enabled servers do.**
+
+**The rule: freeze git-sync on _every_ server (push *and* pull) before rewriting
+shared history.** A pull server mid-rewrite can also resurrect a half-removed
+state. The safe sequence that worked:
+
+1. **Freeze all sync.** `make remote-git-sync-disable-{test,prod}` (flips
+   `enabled: false` in the env-path `git-sync.yaml`).
+2. **Audit where the live secret actually lives — before any `reset --hard`.** A
+   destructive reset *deletes* working-tree files tracked now but absent in the
+   target commit. `config/security-private.php` was such a file — but it was a
+   **stray duplicate**; the authoritative 290-byte copy lives at
+   `env/intotheeast.com/config/security-private.php` (mode 600), which git-sync
+   had copied into `config/`. Because `env/` is gitignored and outside every
+   tracked folder, it survives the reset and *wins* Grav's config merge — so
+   dropping the `config/` copy is safe. **Verify this first** with a secret-safe
+   audit that lists existence + size + `git ls-files` tracking and **never prints
+   contents** (added as `make remote-secrets-audit`; it `ls` / `git ls-files`,
+   never `cat`).
+3. **Force-push `main` to the clean commit.** It sticks now — no server is pushing.
+4. **Reset each server** with `make remote-fetch-content-{test,prod}`
+   (`fetch` → `sparse-checkout disable` → `reset --hard origin/main`). This
+   deletes the stray tracked `config/` copies; the `env/` originals remain.
+5. **Verify** `git ls-files` shows no secret tracked and the `env/` copy is intact
+   on every host.
+6. **Re-enable sync** (`make remote-git-sync-enable-*`), preserving each server's
+   `direction`. Local `HEAD` now equals Gitea `main`, so there is nothing bad to
+   push.
+
+Two gotchas inside step 4:
+
+- **Stale remote-tracking ref.** `reset --hard origin/main` resets to the
+  server's *cached* `refs/remotes/origin/main`, not to Gitea directly. If that ref
+  is stale the reset lands on the wrong commit — confirm the `fetch` force-updated
+  it (`+ f8c45fc...32c3d8c main -> origin/main (forced update)`) before trusting
+  the reset.
+- **`sparse-checkout disable` before `reset --hard`** — otherwise the reset only
+  touches paths inside the sparse pattern and can skip/wipe directories outside it.
+
+**Durable exclusion goes in git-sync's `ignore:` config field, never a
+hand-edited `.gitignore`.** git-sync owns `.gitignore`: on load it regenerates it
+from `folders` (`/*`, `!/pages`, `!/config`, `!/themes`) and **appends** the
+`ignore:` entries. Hand edits are clobbered on the next sync; `ignore:` entries
+persist because git-sync writes them back every time. So the secret paths belong
+in `ignore:` — but that only prevents *future* tracking. The history rewrite
+(steps 1–5) is still required *in addition to* the ignore entries to remove a
+secret that is already committed, not instead of them.
+
 ## Why This Matters
 
 Two quiet, cross-environmental failure modes:
@@ -207,6 +271,9 @@ strength of folder scoping alone.
 - **Reviewing a "stop tracking this secret" cleanup** for whether it will stick:
   is the secret in its own gitignored path (holds) or a line inside a tracked
   functional file that something regenerates (boomerangs)?
+- **Rewriting shared history (force-push, `filter-repo`, `reset --hard`) on a
+  git-sync-managed repo** — freeze sync on every server first, audit where the
+  live secret authoritatively lives before any destructive reset, then re-enable.
 
 ## Examples
 
@@ -229,6 +296,15 @@ per-install and re-commits under sync. The fix that *sticks* is the
 companion-private-file pattern or `popularity.enabled: false` — **not** stripping
 the `salt:` line.
 
+**Force-push revert example.** With `config/security-private.php` +
+`config/versions.yaml` already committed to Gitea `main` (`f8c45fc`), a
+`git push --force origin main` back to the clean `32c3d8c` was undone within
+seconds — prod's `direction: both` git-sync re-pushed its stale `f8c45fc` `HEAD`.
+The rewrite only held after `make remote-git-sync-disable-{test,prod}` froze both
+servers first; then force-push → `make remote-fetch-content-{test,prod}` →
+re-enable. Verified afterward: `git ls-files` on every host lists no secret, and
+each host's `env/…/security-private.php` is intact.
+
 ## Related
 
 - `docs/solutions/conventions/grav-plugin-config-must-be-tracked-override.md` — the