Scope sync to campaigns changed since a given commit

Add SyncChanged (and the ChangedSlugs git-diff helper it's built on) as
an opt-in narrower version of SyncDir: pass the push's "before" SHA and
only the campaign directories that actually changed get looked at at
all, instead of every campaign getting resolved and diffed against
listmonk on every push. An empty or unresolvable since (new branch's
first push, a shallow clone missing that commit, any git failure) always
falls back to a full scan -- this can only ever cause more work than
strictly necessary, never a silently-skipped campaign. SyncDir itself is
now a thin wrapper (SyncChanged with since=""), so every existing caller
and test is unaffected.

campaigns sync PATH now accepts an optional trailing SINCE_SHA.

README.md

@@ -101,11 +101,14 @@ Each file in `attachments:` is uploaded to listmonk's media library under a synt
 
 Every `sync` that actually creates or changes a campaign auto-sends a preview via listmonk's test-send endpoint, to `preview_emails` if set, otherwise `CAMPAIGNS_PREVIEW_EMAIL`. `campaigns test` is for an on-demand re-preview without touching content.
 
+## Sent campaigns stay in the repo
+
+A campaign's `campaign.md` is meant to stick around after it's sent — a record of what went out and when, not something to delete once it's live. Once a campaign is anything other than `draft` in listmonk (`scheduled`/`running`/`paused`/`cancelled`/`finished`), sync leaves it alone entirely — no write, no re-preview, and (as of v0.1.6) no failure either. It's reported as **skipped**, a distinct, non-error bucket from rejected.
+
 ## Sync guardrails
 
 A rejected campaign is usually one of these, all deliberate:
 
-- **Anything other than `draft` in listmonk** (`scheduled`/`running`/`paused`/`cancelled`/`finished`) — sync refuses to touch it once it's live or sent.
 - **A `lists:` name matches zero or more than one listmonk list** — sync never guesses.
 - **An invalid `segment_query`** — listmonk's own error is surfaced verbatim.
 - **Missing `subject`/`from_email`/a target audience, or `type: optin`.**

internal/campaign/sync.go

@@ -6,8 +6,10 @@ import (
 	"encoding/hex"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"sort"
+	"strings"
 
 	"gitea.reground.org/will/eec-campaigns/internal/listmonk"
 )
@@ -18,24 +20,55 @@ import (
 type SyncResult struct {
 	Synced        []string // created or updated (content changed) in listmonk
 	Unchanged     []string // existing draft, content identical — no API write, no preview
-	Rejected      []string // validation/segmentation/list-lookup failure, or non-draft in listmonk — nothing written
+	Skipped       []string // already sent/running/paused/etc. in listmonk — a historical record, left untouched, not an error
+	Rejected      []string // validation/segmentation/list-lookup failure — nothing written
 	PreviewFailed []string // content synced fine, but the automatic preview send itself failed
+	// Mode is purely for CI log visibility into how this run was scoped:
+	// "full scan", or "since <sha>: N of M campaign(s) touched" — see
+	// SyncChanged.
+	Mode string
 }
 
 // SyncDir walks <root>/campaigns/*/campaign.md and upserts each as a
-// listmonk draft campaign. One bad campaign doesn't abort the rest — this
+// listmonk draft campaign. Equivalent to SyncChanged with an empty since —
-// mirrors eec's course.SyncDir/SyncResult shape exactly, extended with the
+// see SyncChanged for the git-diff-scoped version this wraps.
-// Unchanged/PreviewFailed buckets this tool's auto-preview and diff-based
-// sync need. defaultPreviewEmails is used for any campaign that doesn't set
-// its own preview_emails in frontmatter.
 func SyncDir(lm *listmonk.Client, root string, defaultPreviewEmails []string) (*SyncResult, error) {
+	return SyncChanged(lm, root, defaultPreviewEmails, "")
+}
+
+// SyncChanged is SyncDir optionally scoped to just the campaigns whose
+// files changed since a given commit (via ChangedSlugs' git diff), so a
+// push that only touched one campaign doesn't need to fetch and diff every
+// other campaign in the repo against listmonk to confirm nothing changed.
+// An empty since, or ChangedSlugs failing to resolve it for any reason
+// (e.g. a shallow clone missing that commit's history), falls back to a
+// full scan exactly like SyncDir — this can never cause a campaign to be
+// silently skipped, only ever process more than strictly necessary.
+func SyncChanged(lm *listmonk.Client, root string, defaultPreviewEmails []string, since string) (*SyncResult, error) {
 	pattern := filepath.Join(root, "campaigns", "*", "campaign.md")
 	matches, err := filepath.Glob(pattern)
 	if err != nil {
 		return nil, err
 	}
 
-	result := &SyncResult{}
+	mode := "full scan"
+	if since != "" {
+		if slugs, ok := ChangedSlugs(root, since); ok {
+			total := len(matches)
+			filtered := matches[:0]
+			for _, path := range matches {
+				if slugs[filepath.Base(filepath.Dir(path))] {
+					filtered = append(filtered, path)
+				}
+			}
+			matches = filtered
+			mode = fmt.Sprintf("since %s: %d of %d campaign(s) touched", since, len(matches), total)
+		} else {
+			mode = fmt.Sprintf("full scan (could not resolve since %s)", since)
+		}
+	}
+
+	result := &SyncResult{Mode: mode}
 	for _, path := range matches {
 		dir := filepath.Dir(path)
 		slug := filepath.Base(dir)
@@ -45,6 +78,10 @@ func SyncDir(lm *listmonk.Client, root string, defaultPreviewEmails []string) (*
 			result.Rejected = append(result.Rejected, fmt.Sprintf("%s: %v", slug, err))
 			continue
 		}
+		if res.Skipped != "" {
+			result.Skipped = append(result.Skipped, fmt.Sprintf("%s: %s", slug, res.Skipped))
+			continue
+		}
 		if !res.Changed {
 			result.Unchanged = append(result.Unchanged, slug)
 			continue
@@ -57,9 +94,45 @@ func SyncDir(lm *listmonk.Client, root string, defaultPreviewEmails []string) (*
 	return result, nil
 }
 
+// ChangedSlugs returns the set of campaign slugs (directory names under
+// campaigns/) whose files differ between since and HEAD, by shelling out to
+// git diff. The second return is false when since is empty, the all-zero
+// placeholder SHA (what gitea's push event sends for a brand-new branch's
+// first push, since there's no real "before" commit), or the diff itself
+// fails for any reason — most commonly a shallow clone that doesn't have
+// since's history available locally. All of those mean "can't determine
+// what changed," never "nothing changed," so callers must treat false as
+// "fall back to processing everything," not as an empty result.
+func ChangedSlugs(root, since string) (map[string]bool, bool) {
+	if since == "" || since == strings.Repeat("0", 40) {
+		return nil, false
+	}
+	cmd := exec.Command("git", "diff", "--name-only", since, "HEAD", "--", "campaigns")
+	cmd.Dir = root
+	out, err := cmd.Output()
+	if err != nil {
+		return nil, false
+	}
+	slugs := map[string]bool{}
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		if line == "" {
+			continue
+		}
+		// Lines look like "campaigns/<slug>/campaign.md" or
+		// "campaigns/<slug>/assets/whatever.pdf" — the pathspec above
+		// already guarantees the "campaigns/" prefix.
+		parts := strings.SplitN(line, "/", 3)
+		if len(parts) >= 2 {
+			slugs[parts[1]] = true
+		}
+	}
+	return slugs, true
+}
+
 type syncOneResult struct {
 	Changed    bool
-	PreviewErr error // set only when Changed and the automatic preview send failed
+	Skipped    string // non-empty (the listmonk status) when left untouched because it's no longer a draft
+	PreviewErr error  // set only when Changed and the automatic preview send failed
 }
 
 func syncOne(lm *listmonk.Client, slug, dir, path string, defaultPreviewEmails []string) (syncOneResult, error) {
@@ -68,6 +141,22 @@ func syncOne(lm *listmonk.Client, slug, dir, path string, defaultPreviewEmails [
 		return syncOneResult{}, err
 	}
 
+	existing, err := lm.FindCampaignByName(slug)
+	if err != nil {
+		return syncOneResult{}, err
+	}
+	if existing != nil && existing.Status != "draft" {
+		// Once a campaign has actually gone out (or is running/paused/etc.),
+		// its campaign.md is a historical record of what was sent and when —
+		// not something sync should touch or fail the build over. Forcing it
+		// to be deleted just to keep CI green would throw away exactly the
+		// record someone would want to look back at later. Check this before
+		// resolving anything else below, so a since-renamed list or a
+		// since-changed attachment on an old campaign can never break the
+		// build either — an archived campaign has nothing left to resolve.
+		return syncOneResult{Skipped: existing.Status}, nil
+	}
+
 	listIDs, err := resolveLists(lm, fm.Lists)
 	if err != nil {
 		return syncOneResult{}, err
@@ -97,11 +186,6 @@ func syncOne(lm *listmonk.Client, slug, dir, path string, defaultPreviewEmails [
 		MediaIDs:   mediaIDs,
 	}
 
-	existing, err := lm.FindCampaignByName(slug)
-	if err != nil {
-		return syncOneResult{}, err
-	}
-
 	var campaignID int
 	var changed bool
 	switch {
@@ -112,11 +196,6 @@ func syncOne(lm *listmonk.Client, slug, dir, path string, defaultPreviewEmails [
 		}
 		campaignID, changed = created.ID, true
 
-	case existing.Status != "draft":
-		// The core safety invariant: never silently skip or overwrite a
-		// live/sent campaign. This is a hard rejection, not a warning.
-		return syncOneResult{}, fmt.Errorf("campaign is %s in listmonk; sync refuses to modify a non-draft campaign", existing.Status)
-
 	case campaignUnchanged(existing, input):
 		campaignID, changed = existing.ID, false
 

internal/campaign/sync_test.go

@@ -5,6 +5,8 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"os/exec"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -362,6 +364,36 @@ func writeCampaignDir(t *testing.T, root, slug, frontmatterExtra, body string) {
 	writeFile(t, path, "---\n"+frontmatterExtra+"---\n"+body+"\n")
 }
 
+// ---- git test fixtures, for ChangedSlugs/SyncChanged ----
+
+func runGit(t *testing.T, dir string, args ...string) string {
+	t.Helper()
+	cmd := exec.Command("git", args...)
+	cmd.Dir = dir
+	cmd.Env = append(os.Environ(),
+		"GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com",
+		"GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com",
+	)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("git %v: %v\n%s", args, err, out)
+	}
+	return strings.TrimSpace(string(out))
+}
+
+func initGitRepo(t *testing.T, dir string) {
+	t.Helper()
+	runGit(t, dir, "init", "-q", "-b", "master")
+}
+
+// gitCommit stages everything and commits, returning the new commit's SHA.
+func gitCommit(t *testing.T, dir, message string) string {
+	t.Helper()
+	runGit(t, dir, "add", "-A")
+	runGit(t, dir, "commit", "-q", "-m", message)
+	return runGit(t, dir, "rev-parse", "HEAD")
+}
+
 const baseFrontmatter = "subject: \"Big Announcement\"\nlists: [\"Newsletter\"]\nfrom_email: hello@example.com\n"
 
 // ---- tests ----
@@ -479,7 +511,7 @@ func TestSyncDir_UpdatesWhenContentChanges(t *testing.T) {
 	}
 }
 
-func TestSyncDir_RejectsNonDraftCampaign(t *testing.T) {
+func TestSyncDir_SkipsNonDraftCampaignWithoutFailing(t *testing.T) {
 	f := newFakeListmonk(t)
 	f.lists = []fakeList{{ID: 3, Name: "Newsletter"}}
 	f.campaigns = []fakeCampaign{{ID: 1, Name: "launch", Status: "running", Subject: "Original"}}
@@ -492,14 +524,44 @@ func TestSyncDir_RejectsNonDraftCampaign(t *testing.T) {
 	if err != nil {
 		t.Fatalf("SyncDir: %v", err)
 	}
-	if len(result.Rejected) != 1 || !strings.Contains(result.Rejected[0], "running") {
+	// A sent/running campaign is a historical record, not an error -- it
-		t.Fatalf("expected a rejection mentioning 'running', got %v", result.Rejected)
+	// must never land in Rejected (which fails the whole CI job) just
+	// because its campaign.md is still around.
+	if len(result.Rejected) != 0 {
+		t.Fatalf("expected no rejection for an already-sent campaign, got %v", result.Rejected)
+	}
+	if len(result.Skipped) != 1 || !strings.Contains(result.Skipped[0], "running") {
+		t.Fatalf("expected launch in Skipped mentioning 'running', got %v", result.Skipped)
 	}
 	if f.campaigns[0].Subject != "Original" {
 		t.Errorf("expected the live campaign's content to be untouched, got %+v", f.campaigns[0])
 	}
 }
 
+func TestSyncDir_NonDraftCampaignSkipsEvenWithAStaleListReference(t *testing.T) {
+	// A finished campaign's campaign.md might reference a list that's since
+	// been renamed or removed -- that must never break the build, since an
+	// archived campaign has nothing left to resolve.
+	f := newFakeListmonk(t)
+	f.campaigns = []fakeCampaign{{ID: 1, Name: "launch", Status: "finished", Subject: "Original"}}
+	lm := f.client()
+
+	root := t.TempDir()
+	fm := "subject: \"Big Announcement\"\nlists: [\"No Longer Exists\"]\nfrom_email: hello@example.com\n"
+	writeCampaignDir(t, root, "launch", fm, "Hello world.")
+
+	result, err := SyncDir(lm, root, nil)
+	if err != nil {
+		t.Fatalf("SyncDir: %v", err)
+	}
+	if len(result.Rejected) != 0 {
+		t.Fatalf("expected no rejection despite the stale list reference, got %v", result.Rejected)
+	}
+	if len(result.Skipped) != 1 {
+		t.Fatalf("expected launch in Skipped, got %v", result.Skipped)
+	}
+}
+
 func TestSyncDir_AmbiguousListNameRejects(t *testing.T) {
 	f := newFakeListmonk(t)
 	f.lists = []fakeList{{ID: 3, Name: "Newsletter"}, {ID: 4, Name: "Newsletter"}}
@@ -703,3 +765,130 @@ func TestSyncDir_OneBadCampaignDoesNotAbortOthers(t *testing.T) {
 		t.Errorf("expected bad to be rejected with a slug-prefixed message, got %v", result.Rejected)
 	}
 }
+
+// ---- ChangedSlugs / SyncChanged ----
+
+func TestChangedSlugs_ReturnsOnlySlugsTouchedSinceGivenCommit(t *testing.T) {
+	root := t.TempDir()
+	initGitRepo(t, root)
+	writeCampaignDir(t, root, "a", baseFrontmatter, "Original a.")
+	writeCampaignDir(t, root, "b", baseFrontmatter, "Original b.")
+	base := gitCommit(t, root, "add a and b")
+
+	writeCampaignDir(t, root, "a", baseFrontmatter, "Updated a.")
+	gitCommit(t, root, "update a")
+
+	slugs, ok := ChangedSlugs(root, base)
+	if !ok {
+		t.Fatal("expected ChangedSlugs to resolve a valid commit")
+	}
+	if !slugs["a"] || slugs["b"] {
+		t.Errorf("expected only 'a' in changed slugs, got %v", slugs)
+	}
+}
+
+func TestChangedSlugs_AllZeroShaMeansUnresolvable(t *testing.T) {
+	root := t.TempDir()
+	initGitRepo(t, root)
+	writeCampaignDir(t, root, "a", baseFrontmatter, "Hello.")
+	gitCommit(t, root, "add a")
+
+	// The all-zero SHA is what a push event reports as "before" for a
+	// brand-new branch's first push -- there's no real commit to diff
+	// against, so this must signal "can't determine," not "empty diff."
+	_, ok := ChangedSlugs(root, strings.Repeat("0", 40))
+	if ok {
+		t.Error("expected the all-zero SHA to be treated as unresolvable")
+	}
+}
+
+func TestChangedSlugs_UnknownShaMeansUnresolvable(t *testing.T) {
+	root := t.TempDir()
+	initGitRepo(t, root)
+	writeCampaignDir(t, root, "a", baseFrontmatter, "Hello.")
+	gitCommit(t, root, "add a")
+
+	// A commit git has never heard of (e.g. one this shallow clone never
+	// fetched) must fall back to "can't determine," not error out or
+	// silently produce an empty diff.
+	_, ok := ChangedSlugs(root, "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef")
+	if ok {
+		t.Error("expected an unknown commit to be treated as unresolvable")
+	}
+}
+
+func TestChangedSlugs_EmptySinceMeansUnresolvable(t *testing.T) {
+	root := t.TempDir()
+	_, ok := ChangedSlugs(root, "")
+	if ok {
+		t.Error("expected an empty since to be treated as unresolvable")
+	}
+}
+
+func TestSyncChanged_OnlyProcessesCampaignsTouchedSinceGivenCommit(t *testing.T) {
+	f := newFakeListmonk(t)
+	f.lists = []fakeList{{ID: 3, Name: "Newsletter"}}
+	lm := f.client()
+
+	root := t.TempDir()
+	initGitRepo(t, root)
+	writeCampaignDir(t, root, "a", baseFrontmatter, "Original a.")
+	// b's frontmatter references a list that doesn't exist in listmonk --
+	// if SyncChanged ever looks at it, it comes back Rejected.
+	bFrontmatter := "subject: \"B\"\nlists: [\"No Such List\"]\nfrom_email: hello@example.com\n"
+	writeCampaignDir(t, root, "b", bFrontmatter, "Original b.")
+	base := gitCommit(t, root, "add a and b")
+
+	// A full scan at this point must reject b -- sanity check that b's
+	// broken list reference is real, not a mistake in the fixture.
+	sanity, err := SyncDir(lm, root, nil)
+	if err != nil {
+		t.Fatalf("sanity SyncDir: %v", err)
+	}
+	if len(sanity.Rejected) != 1 || !strings.Contains(sanity.Rejected[0], "b:") {
+		t.Fatalf("expected a full scan to reject b, got Rejected=%v", sanity.Rejected)
+	}
+	// Reset: the sanity run may have created campaign "a" in listmonk.
+	f.campaigns = nil
+
+	writeCampaignDir(t, root, "a", baseFrontmatter, "Updated a.")
+	gitCommit(t, root, "update a only")
+
+	result, err := SyncChanged(lm, root, nil, base)
+	if err != nil {
+		t.Fatalf("SyncChanged: %v", err)
+	}
+	if len(result.Synced) != 1 || result.Synced[0] != "a" {
+		t.Errorf("expected only a synced, got Synced=%v", result.Synced)
+	}
+	// The real point of this test: b was never even looked at, despite its
+	// broken list reference, because it wasn't touched by this push.
+	if len(result.Rejected) != 0 {
+		t.Errorf("expected b to be left untouched (not rejected) since it wasn't in the diff, got Rejected=%v", result.Rejected)
+	}
+	if !strings.Contains(result.Mode, "1 of 2") {
+		t.Errorf("expected Mode to report 1 of 2 campaigns touched, got %q", result.Mode)
+	}
+}
+
+func TestSyncChanged_FallsBackToFullScanWhenSinceUnresolvable(t *testing.T) {
+	f := newFakeListmonk(t)
+	f.lists = []fakeList{{ID: 3, Name: "Newsletter"}}
+	lm := f.client()
+
+	root := t.TempDir()
+	initGitRepo(t, root)
+	writeCampaignDir(t, root, "a", baseFrontmatter, "Hello a.")
+	gitCommit(t, root, "add a")
+
+	result, err := SyncChanged(lm, root, nil, "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef")
+	if err != nil {
+		t.Fatalf("SyncChanged: %v", err)
+	}
+	if len(result.Synced) != 1 || result.Synced[0] != "a" {
+		t.Errorf("expected a fallback full scan to still sync a, got Synced=%v", result.Synced)
+	}
+	if !strings.Contains(result.Mode, "full scan") {
+		t.Errorf("expected Mode to note the fallback to a full scan, got %q", result.Mode)
+	}
+}

main.go

@@ -29,10 +29,14 @@ func main() {
 	switch os.Args[1] {
 	case "sync":
 		if len(os.Args) < 3 {
-			fmt.Fprintln(os.Stderr, "usage: campaigns sync PATH")
+			fmt.Fprintln(os.Stderr, "usage: campaigns sync PATH [SINCE_SHA]")
 			os.Exit(1)
 		}
-		runSync(lm, os.Args[2])
+		since := ""
+		if len(os.Args) >= 4 {
+			since = os.Args[3]
+		}
+		runSync(lm, os.Args[2], since)
 	case "send":
 		if len(os.Args) < 3 {
 			fmt.Fprintln(os.Stderr, "usage: campaigns send SLUG")
@@ -65,7 +69,11 @@ func usage() {
 // (comma-separated), used for any campaign that doesn't set its own
 // preview_emails in frontmatter. Optional — sync still works with none set,
 // it just won't auto-preview campaigns that don't specify their own.
-func runSync(lm *listmonk.Client, path string) {
+//
+// since is optional — the commit to diff against (typically the push
+// event's "before" SHA) to scope this run to just the campaigns that
+// changed, per campaign.SyncChanged. Pass "" to always do a full scan.
+func runSync(lm *listmonk.Client, path, since string) {
 	var defaultPreview []string
 	if raw := os.Getenv("CAMPAIGNS_PREVIEW_EMAIL"); raw != "" {
 		for _, addr := range strings.Split(raw, ",") {
@@ -75,12 +83,16 @@ func runSync(lm *listmonk.Client, path string) {
 		}
 	}
 
-	result, err := campaign.SyncDir(lm, path, defaultPreview)
+	result, err := campaign.SyncChanged(lm, path, defaultPreview, since)
 	if err != nil {
 		log.Fatalf("sync: %v", err)
 	}
+	log.Printf("mode: %s", result.Mode)
 	log.Printf("synced: %v", result.Synced)
 	log.Printf("unchanged: %v", result.Unchanged)
+	if len(result.Skipped) > 0 {
+		log.Printf("skipped (already sent): %v", result.Skipped)
+	}
 	if len(result.Rejected) > 0 {
 		log.Printf("rejected: %v", result.Rejected)
 	}