diff --git a/internal/campaign/sync.go b/internal/campaign/sync.go index 08d377c..e201e19 100644 --- a/internal/campaign/sync.go +++ b/internal/campaign/sync.go @@ -6,8 +6,10 @@ import ( "encoding/hex" "fmt" "os" + "os/exec" "path/filepath" "sort" + "strings" "gitea.reground.org/will/eec-campaigns/internal/listmonk" ) @@ -21,22 +23,52 @@ type SyncResult struct { Skipped []string // already sent/running/paused/etc. in listmonk — a historical record, left untouched, not an error Rejected []string // validation/segmentation/list-lookup failure — nothing written PreviewFailed []string // content synced fine, but the automatic preview send itself failed + // Mode is purely for CI log visibility into how this run was scoped: + // "full scan", or "since : N of M campaign(s) touched" — see + // SyncChanged. + Mode string } // SyncDir walks /campaigns/*/campaign.md and upserts each as a -// listmonk draft campaign. One bad campaign doesn't abort the rest — this -// mirrors eec's course.SyncDir/SyncResult shape exactly, extended with the -// Unchanged/PreviewFailed buckets this tool's auto-preview and diff-based -// sync need. defaultPreviewEmails is used for any campaign that doesn't set -// its own preview_emails in frontmatter. +// listmonk draft campaign. Equivalent to SyncChanged with an empty since — +// see SyncChanged for the git-diff-scoped version this wraps. func SyncDir(lm *listmonk.Client, root string, defaultPreviewEmails []string) (*SyncResult, error) { + return SyncChanged(lm, root, defaultPreviewEmails, "") +} + +// SyncChanged is SyncDir optionally scoped to just the campaigns whose +// files changed since a given commit (via ChangedSlugs' git diff), so a +// push that only touched one campaign doesn't need to fetch and diff every +// other campaign in the repo against listmonk to confirm nothing changed. +// An empty since, or ChangedSlugs failing to resolve it for any reason +// (e.g. a shallow clone missing that commit's history), falls back to a +// full scan exactly like SyncDir — this can never cause a campaign to be +// silently skipped, only ever process more than strictly necessary. +func SyncChanged(lm *listmonk.Client, root string, defaultPreviewEmails []string, since string) (*SyncResult, error) { pattern := filepath.Join(root, "campaigns", "*", "campaign.md") matches, err := filepath.Glob(pattern) if err != nil { return nil, err } - result := &SyncResult{} + mode := "full scan" + if since != "" { + if slugs, ok := ChangedSlugs(root, since); ok { + total := len(matches) + filtered := matches[:0] + for _, path := range matches { + if slugs[filepath.Base(filepath.Dir(path))] { + filtered = append(filtered, path) + } + } + matches = filtered + mode = fmt.Sprintf("since %s: %d of %d campaign(s) touched", since, len(matches), total) + } else { + mode = fmt.Sprintf("full scan (could not resolve since %s)", since) + } + } + + result := &SyncResult{Mode: mode} for _, path := range matches { dir := filepath.Dir(path) slug := filepath.Base(dir) @@ -62,6 +94,41 @@ func SyncDir(lm *listmonk.Client, root string, defaultPreviewEmails []string) (* return result, nil } +// ChangedSlugs returns the set of campaign slugs (directory names under +// campaigns/) whose files differ between since and HEAD, by shelling out to +// git diff. The second return is false when since is empty, the all-zero +// placeholder SHA (what gitea's push event sends for a brand-new branch's +// first push, since there's no real "before" commit), or the diff itself +// fails for any reason — most commonly a shallow clone that doesn't have +// since's history available locally. All of those mean "can't determine +// what changed," never "nothing changed," so callers must treat false as +// "fall back to processing everything," not as an empty result. +func ChangedSlugs(root, since string) (map[string]bool, bool) { + if since == "" || since == strings.Repeat("0", 40) { + return nil, false + } + cmd := exec.Command("git", "diff", "--name-only", since, "HEAD", "--", "campaigns") + cmd.Dir = root + out, err := cmd.Output() + if err != nil { + return nil, false + } + slugs := map[string]bool{} + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if line == "" { + continue + } + // Lines look like "campaigns//campaign.md" or + // "campaigns//assets/whatever.pdf" — the pathspec above + // already guarantees the "campaigns/" prefix. + parts := strings.SplitN(line, "/", 3) + if len(parts) >= 2 { + slugs[parts[1]] = true + } + } + return slugs, true +} + type syncOneResult struct { Changed bool Skipped string // non-empty (the listmonk status) when left untouched because it's no longer a draft diff --git a/internal/campaign/sync_test.go b/internal/campaign/sync_test.go index 2f22a9f..4529bd7 100644 --- a/internal/campaign/sync_test.go +++ b/internal/campaign/sync_test.go @@ -5,6 +5,8 @@ import ( "io" "net/http" "net/http/httptest" + "os" + "os/exec" "path/filepath" "strconv" "strings" @@ -362,6 +364,36 @@ func writeCampaignDir(t *testing.T, root, slug, frontmatterExtra, body string) { writeFile(t, path, "---\n"+frontmatterExtra+"---\n"+body+"\n") } +// ---- git test fixtures, for ChangedSlugs/SyncChanged ---- + +func runGit(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + ) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) +} + +func initGitRepo(t *testing.T, dir string) { + t.Helper() + runGit(t, dir, "init", "-q", "-b", "master") +} + +// gitCommit stages everything and commits, returning the new commit's SHA. +func gitCommit(t *testing.T, dir, message string) string { + t.Helper() + runGit(t, dir, "add", "-A") + runGit(t, dir, "commit", "-q", "-m", message) + return runGit(t, dir, "rev-parse", "HEAD") +} + const baseFrontmatter = "subject: \"Big Announcement\"\nlists: [\"Newsletter\"]\nfrom_email: hello@example.com\n" // ---- tests ---- @@ -733,3 +765,130 @@ func TestSyncDir_OneBadCampaignDoesNotAbortOthers(t *testing.T) { t.Errorf("expected bad to be rejected with a slug-prefixed message, got %v", result.Rejected) } } + +// ---- ChangedSlugs / SyncChanged ---- + +func TestChangedSlugs_ReturnsOnlySlugsTouchedSinceGivenCommit(t *testing.T) { + root := t.TempDir() + initGitRepo(t, root) + writeCampaignDir(t, root, "a", baseFrontmatter, "Original a.") + writeCampaignDir(t, root, "b", baseFrontmatter, "Original b.") + base := gitCommit(t, root, "add a and b") + + writeCampaignDir(t, root, "a", baseFrontmatter, "Updated a.") + gitCommit(t, root, "update a") + + slugs, ok := ChangedSlugs(root, base) + if !ok { + t.Fatal("expected ChangedSlugs to resolve a valid commit") + } + if !slugs["a"] || slugs["b"] { + t.Errorf("expected only 'a' in changed slugs, got %v", slugs) + } +} + +func TestChangedSlugs_AllZeroShaMeansUnresolvable(t *testing.T) { + root := t.TempDir() + initGitRepo(t, root) + writeCampaignDir(t, root, "a", baseFrontmatter, "Hello.") + gitCommit(t, root, "add a") + + // The all-zero SHA is what a push event reports as "before" for a + // brand-new branch's first push -- there's no real commit to diff + // against, so this must signal "can't determine," not "empty diff." + _, ok := ChangedSlugs(root, strings.Repeat("0", 40)) + if ok { + t.Error("expected the all-zero SHA to be treated as unresolvable") + } +} + +func TestChangedSlugs_UnknownShaMeansUnresolvable(t *testing.T) { + root := t.TempDir() + initGitRepo(t, root) + writeCampaignDir(t, root, "a", baseFrontmatter, "Hello.") + gitCommit(t, root, "add a") + + // A commit git has never heard of (e.g. one this shallow clone never + // fetched) must fall back to "can't determine," not error out or + // silently produce an empty diff. + _, ok := ChangedSlugs(root, "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef") + if ok { + t.Error("expected an unknown commit to be treated as unresolvable") + } +} + +func TestChangedSlugs_EmptySinceMeansUnresolvable(t *testing.T) { + root := t.TempDir() + _, ok := ChangedSlugs(root, "") + if ok { + t.Error("expected an empty since to be treated as unresolvable") + } +} + +func TestSyncChanged_OnlyProcessesCampaignsTouchedSinceGivenCommit(t *testing.T) { + f := newFakeListmonk(t) + f.lists = []fakeList{{ID: 3, Name: "Newsletter"}} + lm := f.client() + + root := t.TempDir() + initGitRepo(t, root) + writeCampaignDir(t, root, "a", baseFrontmatter, "Original a.") + // b's frontmatter references a list that doesn't exist in listmonk -- + // if SyncChanged ever looks at it, it comes back Rejected. + bFrontmatter := "subject: \"B\"\nlists: [\"No Such List\"]\nfrom_email: hello@example.com\n" + writeCampaignDir(t, root, "b", bFrontmatter, "Original b.") + base := gitCommit(t, root, "add a and b") + + // A full scan at this point must reject b -- sanity check that b's + // broken list reference is real, not a mistake in the fixture. + sanity, err := SyncDir(lm, root, nil) + if err != nil { + t.Fatalf("sanity SyncDir: %v", err) + } + if len(sanity.Rejected) != 1 || !strings.Contains(sanity.Rejected[0], "b:") { + t.Fatalf("expected a full scan to reject b, got Rejected=%v", sanity.Rejected) + } + // Reset: the sanity run may have created campaign "a" in listmonk. + f.campaigns = nil + + writeCampaignDir(t, root, "a", baseFrontmatter, "Updated a.") + gitCommit(t, root, "update a only") + + result, err := SyncChanged(lm, root, nil, base) + if err != nil { + t.Fatalf("SyncChanged: %v", err) + } + if len(result.Synced) != 1 || result.Synced[0] != "a" { + t.Errorf("expected only a synced, got Synced=%v", result.Synced) + } + // The real point of this test: b was never even looked at, despite its + // broken list reference, because it wasn't touched by this push. + if len(result.Rejected) != 0 { + t.Errorf("expected b to be left untouched (not rejected) since it wasn't in the diff, got Rejected=%v", result.Rejected) + } + if !strings.Contains(result.Mode, "1 of 2") { + t.Errorf("expected Mode to report 1 of 2 campaigns touched, got %q", result.Mode) + } +} + +func TestSyncChanged_FallsBackToFullScanWhenSinceUnresolvable(t *testing.T) { + f := newFakeListmonk(t) + f.lists = []fakeList{{ID: 3, Name: "Newsletter"}} + lm := f.client() + + root := t.TempDir() + initGitRepo(t, root) + writeCampaignDir(t, root, "a", baseFrontmatter, "Hello a.") + gitCommit(t, root, "add a") + + result, err := SyncChanged(lm, root, nil, "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef") + if err != nil { + t.Fatalf("SyncChanged: %v", err) + } + if len(result.Synced) != 1 || result.Synced[0] != "a" { + t.Errorf("expected a fallback full scan to still sync a, got Synced=%v", result.Synced) + } + if !strings.Contains(result.Mode, "full scan") { + t.Errorf("expected Mode to note the fallback to a full scan, got %q", result.Mode) + } +} diff --git a/main.go b/main.go index d458ee1..a5013ee 100644 --- a/main.go +++ b/main.go @@ -29,10 +29,14 @@ func main() { switch os.Args[1] { case "sync": if len(os.Args) < 3 { - fmt.Fprintln(os.Stderr, "usage: campaigns sync PATH") + fmt.Fprintln(os.Stderr, "usage: campaigns sync PATH [SINCE_SHA]") os.Exit(1) } - runSync(lm, os.Args[2]) + since := "" + if len(os.Args) >= 4 { + since = os.Args[3] + } + runSync(lm, os.Args[2], since) case "send": if len(os.Args) < 3 { fmt.Fprintln(os.Stderr, "usage: campaigns send SLUG") @@ -65,7 +69,11 @@ func usage() { // (comma-separated), used for any campaign that doesn't set its own // preview_emails in frontmatter. Optional — sync still works with none set, // it just won't auto-preview campaigns that don't specify their own. -func runSync(lm *listmonk.Client, path string) { +// +// since is optional — the commit to diff against (typically the push +// event's "before" SHA) to scope this run to just the campaigns that +// changed, per campaign.SyncChanged. Pass "" to always do a full scan. +func runSync(lm *listmonk.Client, path, since string) { var defaultPreview []string if raw := os.Getenv("CAMPAIGNS_PREVIEW_EMAIL"); raw != "" { for _, addr := range strings.Split(raw, ",") { @@ -75,10 +83,11 @@ func runSync(lm *listmonk.Client, path string) { } } - result, err := campaign.SyncDir(lm, path, defaultPreview) + result, err := campaign.SyncChanged(lm, path, defaultPreview, since) if err != nil { log.Fatalf("sync: %v", err) } + log.Printf("mode: %s", result.Mode) log.Printf("synced: %v", result.Synced) log.Printf("unchanged: %v", result.Unchanged) if len(result.Skipped) > 0 {