1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
|
package persistence
// Regression: when a transaction's apply fails AND its rollback fails, the
// journal is the only durable record of the pre-transaction before-images.
// Deleting it at commit time made the corruption permanent; it must be kept
// so the next Recover call (or operator repair) can retry the rollback.
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestCommitKeepsJournalAfterFailedRollback(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "character.yaml")
if err := os.WriteFile(path, []byte("before"), 0600); err != nil {
t.Fatal(err)
}
journalDir := filepath.Join(dir, "journals")
txn := NewTransaction(journalDir)
if err := txn.Replace(path, []byte("after"), 0600); err != nil {
t.Fatal(err)
}
// Deterministically force both the apply and the rollback writes to
// fail after the snapshot: the target becomes a nonempty directory,
// so WriteFile cannot create the file and rollback cannot restore it.
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
if err := os.Mkdir(path, 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(path, "blocker"), []byte("x"), 0600); err != nil {
t.Fatal(err)
}
err := txn.Commit()
if err == nil || !strings.Contains(err.Error(), "rollback failed") {
t.Fatalf("need an apply+rollback failure, got: %v", err)
}
entries, err := ListJournalFiles(journalDir)
if err != nil {
t.Fatal(err)
}
if len(entries) == 0 {
t.Fatal("failed rollback deleted the only durable before-image; restart cannot recover the pre-transaction state")
}
}
// TestRecoverRetriesKeptRollbackJournal: after the operator repairs the
// fault that broke the rollback, the next Recover completes it.
func TestRecoverRetriesKeptRollbackJournal(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "character.yaml")
if err := os.WriteFile(path, []byte("before"), 0600); err != nil {
t.Fatal(err)
}
journalDir := filepath.Join(dir, "journals")
txn := NewTransaction(journalDir)
if err := txn.Replace(path, []byte("after"), 0600); err != nil {
t.Fatal(err)
}
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
if err := os.Mkdir(path, 0700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(path, "blocker"), []byte("x"), 0600); err != nil {
t.Fatal(err)
}
if err := txn.Commit(); err == nil {
t.Fatal("expected the injected apply/rollback failure")
}
// Operator repair: put the pre-transaction file back.
if err := os.RemoveAll(path); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte("after"), 0600); err != nil {
t.Fatal(err)
}
if err := Recover(journalDir); err != nil {
t.Fatalf("recovery retry failed: %v", err)
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
if string(data) != "before" {
t.Fatalf("recovery did not roll back to the before-image: %q", data)
}
entries, err := ListJournalFiles(journalDir)
if err != nil {
t.Fatal(err)
}
if len(entries) != 0 {
t.Fatalf("completed recovery left journals behind: %v", entries)
}
}
|
