diff options
| -rw-r--r-- | building_guide/concurrency.md | 9 | ||||
| -rw-r--r-- | internal/game/casino.go | 25 | ||||
| -rw-r--r-- | internal/game/economy.go | 51 | ||||
| -rw-r--r-- | internal/game/economy_rollback_regression_test.go | 89 |
4 files changed, 165 insertions, 9 deletions
diff --git a/building_guide/concurrency.md b/building_guide/concurrency.md index 74bbdc7..081952f 100644 --- a/building_guide/concurrency.md +++ b/building_guide/concurrency.md @@ -163,6 +163,15 @@ wallet degrades safely: chip stake is not returned in kind on top of credits that already returned it). If the character save fails too, no evidence exists and the open wager record drives the ordinary refund-on-lost-state recovery. + The same save records the attempted payout record's ID in the character + file's `open_payouts` evidence list. A failed append is not proof that no + record was written — a failed rollback (the truncate, or the sync that + would make the truncation durable) can leave the record bytes behind, and + a retry can leave bytes from an earlier attempt even when the final error + is definite. Recording the ID means a surviving open payout is settled by + recovery from the file evidence instead of being re-applied (a + double-pay); when no record survived, the stale ID is pruned at the next + startup. - A journal that cannot be fully READ (oversized or unreadable line) or that contains a malformed record with later records present aborts recovery and compaction entirely: records are preserved verbatim for diff --git a/internal/game/casino.go b/internal/game/casino.go index 736add2..f111ed0 100644 --- a/internal/game/casino.go +++ b/internal/game/casino.go @@ -136,6 +136,22 @@ func (w casinoWallet) payCreditsFlagged(amount int, refund bool, stakeIDs []stri return capErr } w.p.Credits = creditsAfter + // The append failed, but the record may still be IN the journal: a + // failed rollback (a truncate, or the sync that would make the + // truncation durable) leaves the bytes behind, and appendRetry can + // leave bytes from an earlier attempt even when its final error is + // definite. Record the attempted record's ID as open-payout evidence + // in the SAME save that makes this payout durable, so recovery + // recognizes a surviving record as already paid instead of + // re-applying it (a double-pay). When no record survived, the stale + // ID is pruned at the next startup. + attemptID := "" + if rec != nil { + attemptID = rec.ID + } + if attemptID != "" { + w.p.OpenPayouts = append(w.p.OpenPayouts, attemptID) + } // The character file is the only durable record, so it must also // carry the resolution evidence for the stakes this settlement // resolves: a crash before the games' own ResolveStakes call would @@ -148,6 +164,9 @@ func (w casinoWallet) payCreditsFlagged(amount int, refund bool, stakeIDs []stri resolved := w.recordStakeEvidence(stakeIDs) if serr := w.g.AccountStore.SaveCharacter(w.p); serr != nil { w.p.Credits = creditsBefore + if attemptID != "" { + removeOpenPayout(w.p, attemptID) + } if resolved > 0 { w.p.ResolvedStakes = w.p.ResolvedStakes[:len(w.p.ResolvedStakes)-resolved] } @@ -157,7 +176,11 @@ func (w casinoWallet) payCreditsFlagged(amount int, refund bool, stakeIDs []stri if refund { kind = "refund" } - fmt.Fprintf(os.Stderr, "economy: %s payout of %d credits for %s could not be journaled; the character file is the only durable record\n", kind, amount, w.p.Name) + if errors.Is(err, errJournalAppendUncertain) { + fmt.Fprintf(os.Stderr, "economy: %s payout of %d credits for %s could not be journaled and the append could not be rolled back; recorded as open-payout evidence so recovery will not re-apply it\n", kind, amount, w.p.Name) + } else { + fmt.Fprintf(os.Stderr, "economy: %s payout of %d credits for %s could not be journaled; the character file is the only durable record\n", kind, amount, w.p.Name) + } return nil } if capErr != nil { diff --git a/internal/game/economy.go b/internal/game/economy.go index 9853671..28e2880 100644 --- a/internal/game/economy.go +++ b/internal/game/economy.go @@ -47,6 +47,7 @@ import ( "crypto/rand" "encoding/hex" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -165,8 +166,20 @@ func newEconomyRecordID() string { return hex.EncodeToString(b[:]) } +// errJournalAppendUncertain reports that a journal append failed AND the +// record's bytes could not be proven removed: the rollback truncate failed, +// or the sync that would make that truncation durable failed, or the +// pre-write size was unknown. The record MAY therefore still be present in +// the journal. Callers must not treat the transaction as definitely absent: +// the wallet's journal-unavailable payout fallback records the attempted +// record's ID as open-payout evidence so a surviving record is settled by +// recovery instead of being re-applied (a double-pay). +var errJournalAppendUncertain = errors.New("economy journal append may have left the record in the journal") + // append writes one durable line. Errors are returned to the caller, which -// must treat the transaction as failed. +// must treat the transaction as failed. An error wrapping +// errJournalAppendUncertain additionally means the record may have survived +// in the journal. func (j *economyJournal) append(rec economyRecord) error { rec.TS = time.Now() data, err := json.Marshal(rec) @@ -195,12 +208,20 @@ func (j *economyJournal) append(rec economyRecord) error { startSize = fi.Size() } rollback := func(cause error) error { - if startSize >= 0 { - if terr := f.Truncate(startSize); terr != nil { - fmt.Fprintf(os.Stderr, "economy journal: rollback truncate after %v: %v\n", cause, terr) - } else { - _ = f.Sync() - } + if startSize < 0 { + // The pre-write size is unknown, so the record cannot be truncated + // away with confidence: it may remain in the journal. + return fmt.Errorf("%w: %v (pre-write size unknown)", errJournalAppendUncertain, cause) + } + if terr := f.Truncate(startSize); terr != nil { + fmt.Fprintf(os.Stderr, "economy journal: rollback truncate after %v: %v\n", cause, terr) + return fmt.Errorf("%w: %v (rollback truncate: %v)", errJournalAppendUncertain, cause, terr) + } + if serr := f.Sync(); serr != nil { + // The truncation is not durable: a crash can restore the record + // bytes, so the append outcome is still unknown. + fmt.Fprintf(os.Stderr, "economy journal: rollback sync after %v: %v\n", cause, serr) + return fmt.Errorf("%w: %v (rollback sync: %v)", errJournalAppendUncertain, cause, serr) } return cause } @@ -283,10 +304,20 @@ func (j *economyJournal) repairTornTail() error { // that money correctness depends on. func (j *economyJournal) appendRetry(rec economyRecord, attempts int) error { var err error + uncertain := false for i := 0; i < max(attempts, 1); i++ { if err = j.append(rec); err == nil { return nil } + if errors.Is(err, errJournalAppendUncertain) { + uncertain = true + } + } + // An earlier attempt whose rollback could not be proven may have left the + // record bytes behind even when the final attempt's error is definite, so + // the aggregate must still report that the record MAY be present. + if uncertain && !errors.Is(err, errJournalAppendUncertain) { + err = fmt.Errorf("%w: %v", errJournalAppendUncertain, err) } return err } @@ -324,7 +355,11 @@ func (j *economyJournal) startKind(kind string, refund bool, covers []string, p j.seq++ rec.Seq = j.seq if err := j.appendRetry(*rec, 5); err != nil { - return nil, err + // Return the attempted record with the error. It is NOT durable, but + // its ID is what a surviving (uncertain) append would carry, so the + // wallet's journal-unavailable payout fallback can record it as + // open-payout evidence and stop recovery re-applying the record. + return rec, err } if rec.Tracked { j.openPayouts[rec.ID] = true diff --git a/internal/game/economy_rollback_regression_test.go b/internal/game/economy_rollback_regression_test.go new file mode 100644 index 0000000..bceb351 --- /dev/null +++ b/internal/game/economy_rollback_regression_test.go @@ -0,0 +1,89 @@ +package game + +// Rollback-failure regressions for the durable money journal. When a journal +// append fails, the invariant was historically "append failed means no record +// bytes remain". A failed rollback (the truncate, or the sync that makes it +// durable) breaks that invariant: the record can survive while the caller +// believes the append failed. The wallet's journal-unavailable payout fallback +// must then still record the attempted record's ID as open-payout evidence, or +// recovery re-applies the surviving payout and pays the player twice. + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" +) + +// TestAppendRollbackFailureIsUncertain: /dev/full accepts the open but fails +// the write with ENOSPC and the rollback truncate with EINVAL, so the append +// must report that the record MAY have survived. +func TestAppendRollbackFailureIsUncertain(t *testing.T) { + if _, err := os.Stat("/dev/full"); err != nil { + t.Skipf("/dev/full is not available: %v", err) + } + j := newEconomyJournal(t.TempDir()) + j.path = "/dev/full" + + err := j.append(economyRecord{ID: "uncertain", Kind: "payout", Status: "open"}) + if !errors.Is(err, errJournalAppendUncertain) { + t.Fatalf("append rollback failure = %v, want errJournalAppendUncertain", err) + } +} + +// TestFallbackPayoutRecordsEvidenceForSurvivingRecord: a payout whose journal +// append fails and whose rollback cannot be proven must leave open-payout +// evidence in the character file. A surviving record is then settled from that +// evidence at the next startup instead of being re-applied. +func TestFallbackPayoutRecordsEvidenceForSurvivingRecord(t *testing.T) { + g, p := newEvidenceHarness(t) + + // Break the journal append: point it at its own directory, which cannot + // be opened for append or truncation. The wallet takes the file-only + // fallback. + journalPath := g.Economy.path + g.Economy.path = filepath.Dir(journalPath) + w := casinoWallet{g: g, p: p} + if err := w.PayCredits(20); err != nil { + t.Fatalf("fallback payout failed: %v", err) + } + if p.Credits != 110 { + t.Fatalf("fallback payout not applied: credits=%d, want 110", p.Credits) + } + if len(p.OpenPayouts) != 1 { + t.Fatalf("fallback payout recorded %d open-payout evidence IDs, want 1: %v", + len(p.OpenPayouts), p.OpenPayouts) + } + id := p.OpenPayouts[0] + + // Simulate the failed append having survived in the journal (the exact + // state a failed rollback produces). Recovery must use the file evidence + // and settle it, not re-apply it. + g.Economy.path = journalPath + rec := economyRecord{ + ID: id, Player: p.Name, PlayerID: p.ID, Kind: "payout", Tracked: true, + Credit: 20, Status: "open", CreditsBefore: 90, CreditsAfter: 110, + } + line, err := json.Marshal(rec) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(journalPath, append(line, '\n'), 0o600); err != nil { + t.Fatal(err) + } + + g.recoverEconomy() + + loaded, err := g.AccountStore.LoadCharacter(p.Name) + if err != nil { + t.Fatal(err) + } + if loaded.Credits != 110 { + t.Fatalf("balance after recovery = %d, want 110 (surviving payout settled from evidence, not re-applied)", + loaded.Credits) + } + if open := g.Economy.mustLoadOpen(t); len(open) != 0 { + t.Fatalf("records left open after recovery: %+v", open) + } +} |