Scope sync to campaigns changed since a given commit

Add SyncChanged (and the ChangedSlugs git-diff helper it's built on) as
an opt-in narrower version of SyncDir: pass the push's "before" SHA and
only the campaign directories that actually changed get looked at at
all, instead of every campaign getting resolved and diffed against
listmonk on every push. An empty or unresolvable since (new branch's
first push, a shallow clone missing that commit, any git failure) always
falls back to a full scan -- this can only ever cause more work than
strictly necessary, never a silently-skipped campaign. SyncDir itself is
now a thin wrapper (SyncChanged with since=""), so every existing caller
and test is unaffected.

campaigns sync PATH now accepts an optional trailing SINCE_SHA.

internal/campaign/sync.go

@@ -6,8 +6,10 @@ import (
 	"encoding/hex"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"sort"
+	"strings"
 
 	"gitea.reground.org/will/eec-campaigns/internal/listmonk"
 )
@@ -21,22 +23,52 @@ type SyncResult struct {
 	Skipped       []string // already sent/running/paused/etc. in listmonk — a historical record, left untouched, not an error
 	Rejected      []string // validation/segmentation/list-lookup failure — nothing written
 	PreviewFailed []string // content synced fine, but the automatic preview send itself failed
+	// Mode is purely for CI log visibility into how this run was scoped:
+	// "full scan", or "since <sha>: N of M campaign(s) touched" — see
+	// SyncChanged.
+	Mode string
 }
 
 // SyncDir walks <root>/campaigns/*/campaign.md and upserts each as a
-// listmonk draft campaign. One bad campaign doesn't abort the rest — this
-// mirrors eec's course.SyncDir/SyncResult shape exactly, extended with the
-// Unchanged/PreviewFailed buckets this tool's auto-preview and diff-based
-// sync need. defaultPreviewEmails is used for any campaign that doesn't set
-// its own preview_emails in frontmatter.
+// listmonk draft campaign. Equivalent to SyncChanged with an empty since —
+// see SyncChanged for the git-diff-scoped version this wraps.
 func SyncDir(lm *listmonk.Client, root string, defaultPreviewEmails []string) (*SyncResult, error) {
+	return SyncChanged(lm, root, defaultPreviewEmails, "")
+}
+
+// SyncChanged is SyncDir optionally scoped to just the campaigns whose
+// files changed since a given commit (via ChangedSlugs' git diff), so a
+// push that only touched one campaign doesn't need to fetch and diff every
+// other campaign in the repo against listmonk to confirm nothing changed.
+// An empty since, or ChangedSlugs failing to resolve it for any reason
+// (e.g. a shallow clone missing that commit's history), falls back to a
+// full scan exactly like SyncDir — this can never cause a campaign to be
+// silently skipped, only ever process more than strictly necessary.
+func SyncChanged(lm *listmonk.Client, root string, defaultPreviewEmails []string, since string) (*SyncResult, error) {
 	pattern := filepath.Join(root, "campaigns", "*", "campaign.md")
 	matches, err := filepath.Glob(pattern)
 	if err != nil {
 		return nil, err
 	}
 
-	result := &SyncResult{}
+	mode := "full scan"
+	if since != "" {
+		if slugs, ok := ChangedSlugs(root, since); ok {
+			total := len(matches)
+			filtered := matches[:0]
+			for _, path := range matches {
+				if slugs[filepath.Base(filepath.Dir(path))] {
+					filtered = append(filtered, path)
+				}
+			}
+			matches = filtered
+			mode = fmt.Sprintf("since %s: %d of %d campaign(s) touched", since, len(matches), total)
+		} else {
+			mode = fmt.Sprintf("full scan (could not resolve since %s)", since)
+		}
+	}
+
+	result := &SyncResult{Mode: mode}
 	for _, path := range matches {
 		dir := filepath.Dir(path)
 		slug := filepath.Base(dir)
@@ -62,6 +94,41 @@ func SyncDir(lm *listmonk.Client, root string, defaultPreviewEmails []string) (*
 	return result, nil
 }
 
+// ChangedSlugs returns the set of campaign slugs (directory names under
+// campaigns/) whose files differ between since and HEAD, by shelling out to
+// git diff. The second return is false when since is empty, the all-zero
+// placeholder SHA (what gitea's push event sends for a brand-new branch's
+// first push, since there's no real "before" commit), or the diff itself
+// fails for any reason — most commonly a shallow clone that doesn't have
+// since's history available locally. All of those mean "can't determine
+// what changed," never "nothing changed," so callers must treat false as
+// "fall back to processing everything," not as an empty result.
+func ChangedSlugs(root, since string) (map[string]bool, bool) {
+	if since == "" || since == strings.Repeat("0", 40) {
+		return nil, false
+	}
+	cmd := exec.Command("git", "diff", "--name-only", since, "HEAD", "--", "campaigns")
+	cmd.Dir = root
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, false
+	}
+	slugs := map[string]bool{}
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		if line == "" {
+			continue
+		}
+		// Lines look like "campaigns/<slug>/campaign.md" or
+		// "campaigns/<slug>/assets/whatever.pdf" — the pathspec above
+		// already guarantees the "campaigns/" prefix.
+		parts := strings.SplitN(line, "/", 3)
+		if len(parts) >= 2 {
+			slugs[parts[1]] = true
+		}
+	}
+	return slugs, true
+}
+
 type syncOneResult struct {
 	Changed    bool
 	Skipped    string // non-empty (the listmonk status) when left untouched because it's no longer a draft

internal/campaign/sync_test.go

@@ -5,6 +5,8 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"os/exec"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -362,6 +364,36 @@ func writeCampaignDir(t *testing.T, root, slug, frontmatterExtra, body string) {
 	writeFile(t, path, "---\n"+frontmatterExtra+"---\n"+body+"\n")
 }
 
+// ---- git test fixtures, for ChangedSlugs/SyncChanged ----
+
+func runGit(t *testing.T, dir string, args ...string) string {
+	t.Helper()
+	cmd := exec.Command("git", args...)
+	cmd.Dir = dir
+	cmd.Env = append(os.Environ(),
+		"GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com",
+		"GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com",
+	)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("git %v: %v\n%s", args, err, out)
+	}
+	return strings.TrimSpace(string(out))
+}
+
+func initGitRepo(t *testing.T, dir string) {
+	t.Helper()
+	runGit(t, dir, "init", "-q", "-b", "master")
+}
+
+// gitCommit stages everything and commits, returning the new commit's SHA.
+func gitCommit(t *testing.T, dir, message string) string {
+	t.Helper()
+	runGit(t, dir, "add", "-A")
+	runGit(t, dir, "commit", "-q", "-m", message)
+	return runGit(t, dir, "rev-parse", "HEAD")
+}
+
 const baseFrontmatter = "subject: \"Big Announcement\"\nlists: [\"Newsletter\"]\nfrom_email: hello@example.com\n"
 
 // ---- tests ----
@@ -733,3 +765,130 @@ func TestSyncDir_OneBadCampaignDoesNotAbortOthers(t *testing.T) {
 		t.Errorf("expected bad to be rejected with a slug-prefixed message, got %v", result.Rejected)
 	}
 }
+
+// ---- ChangedSlugs / SyncChanged ----
+
+func TestChangedSlugs_ReturnsOnlySlugsTouchedSinceGivenCommit(t *testing.T) {
+	root := t.TempDir()
+	initGitRepo(t, root)
+	writeCampaignDir(t, root, "a", baseFrontmatter, "Original a.")
+	writeCampaignDir(t, root, "b", baseFrontmatter, "Original b.")
+	base := gitCommit(t, root, "add a and b")
+
+	writeCampaignDir(t, root, "a", baseFrontmatter, "Updated a.")
+	gitCommit(t, root, "update a")
+
+	slugs, ok := ChangedSlugs(root, base)
+	if !ok {
+		t.Fatal("expected ChangedSlugs to resolve a valid commit")
+	}
+	if !slugs["a"] || slugs["b"] {
+		t.Errorf("expected only 'a' in changed slugs, got %v", slugs)
+	}
+}
+
+func TestChangedSlugs_AllZeroShaMeansUnresolvable(t *testing.T) {
+	root := t.TempDir()
+	initGitRepo(t, root)
+	writeCampaignDir(t, root, "a", baseFrontmatter, "Hello.")
+	gitCommit(t, root, "add a")
+
+	// The all-zero SHA is what a push event reports as "before" for a
+	// brand-new branch's first push -- there's no real commit to diff
+	// against, so this must signal "can't determine," not "empty diff."
+	_, ok := ChangedSlugs(root, strings.Repeat("0", 40))
+	if ok {
+		t.Error("expected the all-zero SHA to be treated as unresolvable")
+	}
+}
+
+func TestChangedSlugs_UnknownShaMeansUnresolvable(t *testing.T) {
+	root := t.TempDir()
+	initGitRepo(t, root)
+	writeCampaignDir(t, root, "a", baseFrontmatter, "Hello.")
+	gitCommit(t, root, "add a")
+
+	// A commit git has never heard of (e.g. one this shallow clone never
+	// fetched) must fall back to "can't determine," not error out or
+	// silently produce an empty diff.
+	_, ok := ChangedSlugs(root, "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef")
+	if ok {
+		t.Error("expected an unknown commit to be treated as unresolvable")
+	}
+}
+
+func TestChangedSlugs_EmptySinceMeansUnresolvable(t *testing.T) {
+	root := t.TempDir()
+	_, ok := ChangedSlugs(root, "")
+	if ok {
+		t.Error("expected an empty since to be treated as unresolvable")
+	}
+}
+
+func TestSyncChanged_OnlyProcessesCampaignsTouchedSinceGivenCommit(t *testing.T) {
+	f := newFakeListmonk(t)
+	f.lists = []fakeList{{ID: 3, Name: "Newsletter"}}
+	lm := f.client()
+
+	root := t.TempDir()
+	initGitRepo(t, root)
+	writeCampaignDir(t, root, "a", baseFrontmatter, "Original a.")
+	// b's frontmatter references a list that doesn't exist in listmonk --
+	// if SyncChanged ever looks at it, it comes back Rejected.
+	bFrontmatter := "subject: \"B\"\nlists: [\"No Such List\"]\nfrom_email: hello@example.com\n"
+	writeCampaignDir(t, root, "b", bFrontmatter, "Original b.")
+	base := gitCommit(t, root, "add a and b")
+
+	// A full scan at this point must reject b -- sanity check that b's
+	// broken list reference is real, not a mistake in the fixture.
+	sanity, err := SyncDir(lm, root, nil)
+	if err != nil {
+		t.Fatalf("sanity SyncDir: %v", err)
+	}
+	if len(sanity.Rejected) != 1 || !strings.Contains(sanity.Rejected[0], "b:") {
+		t.Fatalf("expected a full scan to reject b, got Rejected=%v", sanity.Rejected)
+	}
+	// Reset: the sanity run may have created campaign "a" in listmonk.
+	f.campaigns = nil
+
+	writeCampaignDir(t, root, "a", baseFrontmatter, "Updated a.")
+	gitCommit(t, root, "update a only")
+
+	result, err := SyncChanged(lm, root, nil, base)
+	if err != nil {
+		t.Fatalf("SyncChanged: %v", err)
+	}
+	if len(result.Synced) != 1 || result.Synced[0] != "a" {
+		t.Errorf("expected only a synced, got Synced=%v", result.Synced)
+	}
+	// The real point of this test: b was never even looked at, despite its
+	// broken list reference, because it wasn't touched by this push.
+	if len(result.Rejected) != 0 {
+		t.Errorf("expected b to be left untouched (not rejected) since it wasn't in the diff, got Rejected=%v", result.Rejected)
+	}
+	if !strings.Contains(result.Mode, "1 of 2") {
+		t.Errorf("expected Mode to report 1 of 2 campaigns touched, got %q", result.Mode)
+	}
+}
+
+func TestSyncChanged_FallsBackToFullScanWhenSinceUnresolvable(t *testing.T) {
+	f := newFakeListmonk(t)
+	f.lists = []fakeList{{ID: 3, Name: "Newsletter"}}
+	lm := f.client()
+
+	root := t.TempDir()
+	initGitRepo(t, root)
+	writeCampaignDir(t, root, "a", baseFrontmatter, "Hello a.")
+	gitCommit(t, root, "add a")
+
+	result, err := SyncChanged(lm, root, nil, "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef")
+	if err != nil {
+		t.Fatalf("SyncChanged: %v", err)
+	}
+	if len(result.Synced) != 1 || result.Synced[0] != "a" {
+		t.Errorf("expected a fallback full scan to still sync a, got Synced=%v", result.Synced)
+	}
+	if !strings.Contains(result.Mode, "full scan") {
+		t.Errorf("expected Mode to note the fallback to a full scan, got %q", result.Mode)
+	}
+}

main.go

@@ -29,10 +29,14 @@ func main() {
 	switch os.Args[1] {
 	case "sync":
 		if len(os.Args) < 3 {
-			fmt.Fprintln(os.Stderr, "usage: campaigns sync PATH")
+			fmt.Fprintln(os.Stderr, "usage: campaigns sync PATH [SINCE_SHA]")
 			os.Exit(1)
 		}
-		runSync(lm, os.Args[2])
+		since := ""
+		if len(os.Args) >= 4 {
+			since = os.Args[3]
+		}
+		runSync(lm, os.Args[2], since)
 	case "send":
 		if len(os.Args) < 3 {
 			fmt.Fprintln(os.Stderr, "usage: campaigns send SLUG")
@@ -65,7 +69,11 @@ func usage() {
 // (comma-separated), used for any campaign that doesn't set its own
 // preview_emails in frontmatter. Optional — sync still works with none set,
 // it just won't auto-preview campaigns that don't specify their own.
-func runSync(lm *listmonk.Client, path string) {
+//
+// since is optional — the commit to diff against (typically the push
+// event's "before" SHA) to scope this run to just the campaigns that
+// changed, per campaign.SyncChanged. Pass "" to always do a full scan.
+func runSync(lm *listmonk.Client, path, since string) {
 	var defaultPreview []string
 	if raw := os.Getenv("CAMPAIGNS_PREVIEW_EMAIL"); raw != "" {
 		for _, addr := range strings.Split(raw, ",") {
@@ -75,10 +83,11 @@ func runSync(lm *listmonk.Client, path string) {
 		}
 	}
 
-	result, err := campaign.SyncDir(lm, path, defaultPreview)
+	result, err := campaign.SyncChanged(lm, path, defaultPreview, since)
 	if err != nil {
 		log.Fatalf("sync: %v", err)
 	}
+	log.Printf("mode: %s", result.Mode)
 	log.Printf("synced: %v", result.Synced)
 	log.Printf("unchanged: %v", result.Unchanged)
 	if len(result.Skipped) > 0 {