diff --git a/cmd/bdbench/live.html b/cmd/bdbench/live.html index a992fea..f789bd6 100644 --- a/cmd/bdbench/live.html +++ b/cmd/bdbench/live.html @@ -2,7 +2,12 @@

bdbench live

-
refreshes every 5 s from bdbench.csv
-
+
live state every 2 s (live.json); a report row every minute (bdbench.csv)
+
+
+

seal p50 / p90 / max (ms) — must stay flat with age

block p50 / p90 / max (ms) — interval is 1000

@@ -28,16 +35,30 @@

bdbench live

function line(id,rows,keys,colors,ymax){const c=document.getElementById(id);const W=c.width=c.clientWidth*2,H=c.height=c.clientHeight*2;const x=c.getContext("2d");x.clearRect(0,0,W,H);if(!rows.length)return; let m=ymax||Math.max(1,...keys.flatMap(k=>rows.map(r=>r[k]||0)))*1.1;x.strokeStyle="#2a3240";x.lineWidth=1;for(let g=0;g<=4;g++){const y=H-6-(H-12)*g/4;x.beginPath();x.moveTo(40,y);x.lineTo(W,y);x.stroke();x.fillStyle="#8b949e";x.font="18px sans-serif";x.fillText((m*g/4).toFixed(0),0,y+6)} keys.forEach((k,i)=>{x.strokeStyle=colors[i];x.lineWidth=3;x.beginPath();rows.forEach((r,j)=>{const px=40+(W-44)*j/Math.max(1,rows.length-1),py=H-6-(H-12)*Math.min(1,(r[k]||0)/m);j?x.lineTo(px,py):x.moveTo(px,py)});x.stroke()})} +async function now(){try{const l=await (await fetch("live.json?"+Date.now())).json();const m=Math.floor(l.elapsedSec/60),s=l.elapsedSec%60;document.getElementById("state").textContent=`live · ${m}m ${String(s).padStart(2,"0")}s · block ${l.height} · ${l.blocks} blocks`;const L=l.last10s; + document.getElementById("now").innerHTML=[ + group("Now: seals, last 10 s",[["p50",L.sealP50ms.toFixed(0)+" ms","big"],["p90 (budget 100)",L.sealP90ms.toFixed(0)+" ms",L.sealP90ms>100?"warn":"ok"],["max",L.sealMaxMs.toFixed(0)+" ms"]]), + group("Now: blocks, last 10 s",[["p50",L.blockP50ms.toFixed(0)+" ms","big"],["p90 (interval 1000)",L.blockP90ms.toFixed(0)+" ms",L.blockP90ms>1000?"bad":"ok"],["max",L.blockMaxMs.toFixed(0)+" ms"],["over the interval",L.over+" of "+L.blocks,L.over>0?"bad":"ok"]]), + group("Now: maintenance and heap",[["passes in flight",l.maintenanceInFlight,"big"],["heap live / dead",l.heapLiveMB.toFixed(0)+" / "+l.heapHoleMB.toFixed(0)+" MB"],["cleaner scanned / copied",l.heapScannedMB.toFixed(0)+" / "+l.heapMovedMB.toFixed(0)+" MB"],["shard sync: heap fsync / delta",l.heapFsyncMsAvg.toFixed(1)+" / "+l.heapDeltaMsAvg.toFixed(1)+" ms avg"],["shard sync covers",l.heapSyncKBAvg.toFixed(0)+" KB avg"],["perm merges / folds / packs",l.permMerges+" / "+l.permFolds+" / "+l.permPacks],["perm index written",(l.permIndexMB||0).toFixed(0)+" MB"],["wrong answers so far",l.mismatches,l.mismatches>0?"bad":"ok"]])].join("")}catch(e){}} async function tick(){let t;try{t=await (await fetch("bdbench.csv?"+Date.now())).text()}catch(e){document.getElementById("state").textContent="unreachable";return} const rows=parse(t),last=rows[rows.length-1]||{};const st=document.getElementById("state"); - st.textContent=rows.length?("minute "+last.minute):"waiting for the first minute"; - const cards=[["seal p90",last.seal_p90_ms+" ms","budget 100"],["block p90",last.block_p90_ms+" ms","interval 1000"],["blocks over interval",last.over_budget,"this minute"],["read p99",last.read_p99_us+" µs",""],["put p99 dyna/perm",last.dyna_put_p99_us+"/"+last.perm_put_p99_us+" µs",""],["maintenance",(+last.compress_s+ +last.merge_s+ +last.pack_s).toFixed(1)+" s/min",last.compress_passes+" passes, "+last.skipped+" skipped"],["disk write",last.write_MBps+" MB/s","read "+last.read_MBps],["store",(last.store_MB/1000).toFixed(2)+" GB",last.files+" files"],["history segments","perm "+last.perm_history+" dyna "+last.dyna_history,"bloom "+last.bloom_MB+" MB"],["wrong answers",last.mismatches,""]]; - document.getElementById("cards").innerHTML=cards.map(c=>`
${c[0]}
0?'class=bad':''}>${c[1]}
${c[2]}
`).join(""); + if(!rows.length)st.textContent="waiting for the first minute"; + const tot=(+last.compress_s+ +last.merge_s+ +last.pack_s).toFixed(1); + document.getElementById("cards").innerHTML=[ + group("Protocol path, this minute",[["seal p90 (budget 100 ms)",last.seal_p90_ms+" ms",last.seal_p90_ms>100?"big warn":"big ok"],["block p90 (interval 1000 ms)",last.block_p90_ms+" ms",last.block_p90_ms>1000?"big bad":"big"],["blocks over the interval",last.over_budget,last.over_budget>0?"bad":"ok"],["blocks this minute",last.blocks],["read p99",last.read_p99_us+" µs"],["put p99 dyna / perm",last.dyna_put_p99_us+" / "+last.perm_put_p99_us+" µs"]]), + group("Maintenance, this minute",[["seconds of work",tot+" s","big"],["compress passes / s",last.compress_passes+" / "+last.compress_s],["merge passes / s",last.merge_passes+" / "+last.merge_s],["pack passes / s",last.pack_passes+" / "+last.pack_s],["skipped (one in flight)",last.skipped]]), + group("Disk",[["write",last.write_MBps+" MB/s","big"],["read",last.read_MBps+" MB/s"],["store size",(last.store_MB/1000).toFixed(2)+" GB"],["files",last.files]]), + group("Store",[["history segments perm / dyna",last.perm_history+" / "+last.dyna_history],["resident filters",last.bloom_MB+" MB"],["wrong answers",last.mismatches,last.mismatches>0?"big bad":"big ok"]])].join(""); line("c_seal",rows,["seal_p50_ms","seal_p90_ms","seal_max_ms"],["#3fb950","#d29922","#f85149"]); line("c_block",rows,["block_p50_ms","block_p90_ms","block_max_ms"],["#3fb950","#d29922","#f85149"],1000); line("c_maint",rows,["compress_s","merge_s","pack_s"],["#58a6ff","#bc8cff","#39c5cf"]); line("c_disk",rows.map(r=>({w:r.write_MBps,g:r.store_MB/1000*10})),["w","g"],["#d29922","#58a6ff"]); const T=document.getElementById("t");T.innerHTML=""+cols.map(c=>""+c.replace(/_/g," ")+"").join("")+""+rows.slice().reverse().map(r=>""+cols.map(c=>{let cls="";if(c=="seal_p90_ms"&&r[c]>100)cls="warn";if(c=="over_budget"&&r[c]>0)cls="bad";if(c=="mismatches"&&r[c]>0)cls="bad";return `${r[c]}`}).join("")+"").join("")} -fetch("run.json").then(r=>r.json()).then(c=>{document.getElementById("sub").textContent=`${c.stores} store(s) × ${c.shards} shards, seal limit ${c.sealLimit}, window ${c.window}, maintenance every ${c.compressEvery} blocks, pack every ${c.packEvery} · per ${c.interval} block: ${c.dynaPuts} dyna puts, ${c.permPuts} perm puts, ${c.reads} reads over ${c.hotKeys} hot keys, values ${c.valueMin}-${c.valueMax} B · ${c.dir} · started ${c.started} for ${c.duration} · refreshes every 5 s from bdbench.csv`}).catch(()=>{}); -tick();setInterval(tick,5000); +function group(title,rows){return `

${title}

${rows.map(r=>``).join("")}
${r[0]}${r[1]}
`} +fetch("run.json").then(r=>r.json()).then(c=>{document.getElementById("config").innerHTML=[ + group("Store",[["dynamic layer",c.dynaHeap?"heap with holes":"sealed segments"],["stores on the disk",c.stores],["shards per store",c.shards],["seal limit (records/shard layer)",c.sealLimit],["window (blocks)",c.window]]), + group("Load per block per store",[["dynamic puts",c.dynaPuts],["permanent puts",c.permPuts],["reads",c.reads],["hot dynamic keys",c.hotKeys],["value bytes",c.valueMin+" to "+c.valueMax]]), + group("Schedule",[["block interval",c.interval],["duration",c.duration],["maintenance every (blocks)",c.compressEvery],["pack every (blocks)",c.packEvery],["started (UTC)",c.started]]), + group("Run",[["directory",c.dir.split("/").slice(-1)[0]],["seed",c.seed]])].join("")}).catch(()=>{}); +now();tick();setInterval(now,2000);setInterval(tick,5000); diff --git a/cmd/bdbench/main.go b/cmd/bdbench/main.go index 643f6e0..f0e7d46 100644 --- a/cmd/bdbench/main.go +++ b/cmd/bdbench/main.go @@ -20,6 +20,7 @@ package main import ( + "bytes" _ "embed" "encoding/csv" "encoding/json" @@ -64,6 +65,9 @@ type config struct { seed uint64 pprof string http string + dynaHeap bool + permFiles bool + phase bool } //go:embed live.html @@ -91,6 +95,9 @@ func parseFlags() (config, error) { flag.Uint64Var(&c.seed, "seed", 1, "random seed") flag.StringVar(&c.pprof, "pprof", "", "serve net/http/pprof on this address (e.g. 127.0.0.1:6061)") flag.StringVar(&c.http, "http", "127.0.0.1:8098", "serve the live page and the run's files here; empty disables") + flag.BoolVar(&c.dynaHeap, "dyna-heap", false, "dynamic layer as a heap with holes (proposal 2026-09-16) instead of sealed segments") + flag.BoolVar(&c.permFiles, "perm-files", false, "both layers as files of entries: the heap and the permanent layer with buckets of keys (implies -dyna-heap)") + flag.BoolVar(&c.phase, "maintenance-phase", false, "offset each store's maintenance cadence by its share of the period, so stores in lockstep do not all maintain at once") flag.Parse() if flag.NArg() != 0 { return c, fmt.Errorf("unexpected arguments: %q", flag.Args()) @@ -129,6 +136,15 @@ func (s *samples) add(d time.Duration) { s.mu.Unlock() } +// addAll takes a block's worth of samples under one lock: forty +// thousand reads a block, nine stores, made the recorder's lock a +// fifth of the benchmark's CPU. +func (s *samples) addAll(v []time.Duration) { + s.mu.Lock() + s.v = append(s.v, v...) + s.mu.Unlock() +} + func (s *samples) take() []time.Duration { s.mu.Lock() v := s.v @@ -149,10 +165,23 @@ func pct(sorted []time.Duration, p float64) time.Duration { func ms(d time.Duration) string { return strconv.FormatFloat(float64(d)/1e6, 'f', 1, 64) } func us(d time.Duration) string { return strconv.FormatInt(int64(d/time.Microsecond), 10) } +// recent is one block as the live page sees it: when it ended and +// what it and its seal cost. +type recent struct { + at time.Time + block, seal time.Duration + store int +} + // tallies is what every store adds to and the report takes from. type tallies struct { blockTimes, sealTimes, dynaPut, permPut, readT samples blocks, over, mismatches atomic.Uint64 + inFlight atomic.Int64 // Maintenance passes running now + + ringMu sync.Mutex + ring [1024]recent // The last blocks, for the live state + ringN uint64 mu sync.Mutex passes map[string]int @@ -188,10 +217,13 @@ type store struct { // A few hot keys are checked on read: the value the store returns // must be the value last written. A platform that only times // answers cannot tell a fast wrong answer from a fast right one. - last map[[32]byte][]byte - height uint64 - maintaining atomic.Bool - maintWG sync.WaitGroup + last map[[32]byte][]byte + height uint64 + phase uint64 // Blocks this store's maintenance cadence is offset by + // A block's samples, handed to the tallies once per block + dynaT, permT, readT []time.Duration + maintaining atomic.Bool + maintWG sync.WaitGroup } const ( @@ -201,7 +233,14 @@ const ( func openStore(c config, id int) (*store, error) { dir := filepath.Join(c.dir, fmt.Sprintf("store-%d", id)) - kv, err := blockchainDB.NewKVShardN(dir, c.shards, c.sealLimit) + open := blockchainDB.NewKVShardN + if c.dynaHeap { + open = blockchainDB.NewKVShardHeapN + } + if c.permFiles { + open = blockchainDB.NewKVShardFilesN + } + kv, err := open(dir, c.shards, c.sealLimit) if err != nil { return nil, fmt.Errorf("open store %d: %w", id, err) } @@ -215,6 +254,9 @@ func openStore(c config, id int) (*store, error) { } s := &store{id: id, kv: kv, rnd: blockchainDB.NewFastRandom(seed), hot: make([][32]byte, c.hotKeys), permKeys: make([][32]byte, 0, permSample), last: make(map[[32]byte][]byte, checked)} + if c.phase && c.compressEvery > 0 { + s.phase = uint64(id) * c.compressEvery / uint64(c.stores) + } // Hot dynamic keys are rewritten with a skew (index = hot * r^2, so // the low indexes take most writes); permanent keys are always new, // and a bounded sample of them, across all ages, is what the @@ -228,6 +270,12 @@ func openStore(c config, id int) (*store, error) { func (s *store) block(c config, t *tallies) error { s.height++ start := time.Now() + s.dynaT, s.permT, s.readT = s.dynaT[:0], s.permT[:0], s.readT[:0] + defer func() { + t.dynaPut.addAll(s.dynaT) + t.permPut.addAll(s.permT) + t.readT.addAll(s.readT) + }() for i := 0; i < c.dynaPuts; i++ { r := float64(s.rnd.UintN(1<<20)) / (1 << 20) k := s.hot[int(r*r*float64(c.hotKeys))%c.hotKeys] @@ -236,19 +284,19 @@ func (s *store) block(c config, t *tallies) error { if err := s.kv.PutDyna(k, v); err != nil { return fmt.Errorf("store %d PutDyna: %w", s.id, err) } - t.dynaPut.add(time.Since(at)) + s.dynaT = append(s.dynaT, time.Since(at)) if len(s.last) < checked || s.last[k] != nil { s.last[k] = v } } for i := 0; i < c.permPuts; i++ { k := s.rnd.NextHash() - v := s.rnd.RandBuff(c.valueMin, c.valueMax) + v := permValue(k, c.valueMin, c.valueMax) at := time.Now() if err := s.kv.PutPerm(k, v); err != nil { return fmt.Errorf("store %d PutPerm: %w", s.id, err) } - t.permPut.add(time.Since(at)) + s.permT = append(s.permT, time.Since(at)) if len(s.permKeys) < permSample { s.permKeys = append(s.permKeys, k) } else if s.rnd.UintN(64) == 0 { // Keep the sample spread across every age @@ -259,20 +307,32 @@ func (s *store) block(c config, t *tallies) error { for i := 0; i < c.reads; i++ { var k [32]byte var get func([32]byte) ([]byte, error) + perm := false switch pick := s.rnd.UintN(6); { case pick < 3: k, get = s.hot[int(s.rnd.UintN(uint(c.hotKeys)))], s.kv.GetDyna case pick < 5 && len(s.permKeys) > 0: - k, get = s.permKeys[int(s.rnd.UintN(uint(len(s.permKeys))))], s.kv.GetPerm + // A permanent key of any age, through the deep read the + // adapter uses for anything older than the window (Get + // answers the permanent layer's window only, by design) + k, get, perm = s.permKeys[int(s.rnd.UintN(uint(len(s.permKeys))))], s.kv.GetDeep, true default: k, get = s.rnd.NextHash(), s.kv.Get } at := time.Now() v, err := get(k) - t.readT.add(time.Since(at)) + s.readT = append(s.readT, time.Since(at)) if err != nil && !errors.Is(err, os.ErrNotExist) && !strings.Contains(err.Error(), "not found") { return fmt.Errorf("store %d read: %w", s.id, err) } + // A permanent key that was written must come back, with the + // value its key derives; a checked hot key must come back as + // last written. A platform that only times answers cannot + // tell a fast wrong answer, or a fast "not found", from a + // right one. + if perm && (err != nil || !bytes.Equal(v, permValue(k, c.valueMin, c.valueMax))) { + t.mismatches.Add(1) + } if want, ok := s.last[k]; ok && err == nil && string(v) != string(want) { t.mismatches.Add(1) } @@ -281,16 +341,21 @@ func (s *store) block(c config, t *tallies) error { if err := s.kv.SealBlock(s.height); err != nil { return fmt.Errorf("store %d SealBlock: %w", s.id, err) } - t.sealTimes.add(time.Since(at)) + sealTook := time.Since(at) + t.sealTimes.add(sealTook) took := time.Since(start) t.blockTimes.add(took) t.blocks.Add(1) + t.ringMu.Lock() + t.ring[t.ringN%uint64(len(t.ring))] = recent{at: time.Now(), block: took, seal: sealTook, store: s.id} + t.ringN++ + t.ringMu.Unlock() if took > c.interval { t.over.Add(1) } else { time.Sleep(c.interval - took) } - if c.compressEvery > 0 && s.height%c.compressEvery == 0 { + if c.compressEvery > 0 && (s.height+s.phase)%c.compressEvery == 0 { s.maintain(c, t) } return nil @@ -307,9 +372,11 @@ func (s *store) maintain(c config, t *tallies) { } height := s.height s.maintWG.Add(1) + t.inFlight.Add(1) go func() { defer s.maintWG.Done() defer s.maintaining.Store(false) + defer t.inFlight.Add(-1) at := time.Now() err := s.kv.Compress() t.note("compress", time.Since(at)) @@ -353,6 +420,8 @@ func procIO() (read, write uint64) { return } +// dirSize counts the bytes the files occupy, not their apparent size: +// a heap releases regions with punched holes and keeps its length. func dirSize(dir string) (files int, bytes int64) { _ = filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error { if err != nil || d.IsDir() { @@ -360,13 +429,140 @@ func dirSize(dir string) (files int, bytes int64) { } if info, err := d.Info(); err == nil { files++ - bytes += info.Size() + if st, ok := info.Sys().(*syscall.Stat_t); ok { + bytes += st.Blocks * 512 + } else { + bytes += info.Size() + } } return nil }) return } +// liveState is what the page shows between report rows: the last ten +// seconds of blocks and seals, and the run's running totals. Written +// to live.json every two seconds. +func (t *tallies) liveState(c config, stores []*store, start time.Time) []byte { + t.ringMu.Lock() + cut := time.Now().Add(-10 * time.Second) + var bt, st []time.Duration + var perStore [][]time.Duration + n := t.ringN + if n > uint64(len(t.ring)) { + n = uint64(len(t.ring)) + } + for i := uint64(0); i < n; i++ { + r := t.ring[(t.ringN-1-i)%uint64(len(t.ring))] + if r.at.Before(cut) { + break + } + bt = append(bt, r.block) + st = append(st, r.seal) + for len(perStore) <= r.store { + perStore = append(perStore, nil) + } + perStore[r.store] = append(perStore[r.store], r.seal) + } + t.ringMu.Unlock() + // Each store's seal p90 over the same window: a tail that is one + // store's looks different from a tail every store shares + storeP90 := make([]float64, len(perStore)) + for i, v := range perStore { + sort.Slice(v, func(a, b int) bool { return v[a] < v[b] }) + storeP90[i] = float64(pct(v, .9)) / 1e6 + } + sort.Slice(bt, func(i, j int) bool { return bt[i] < bt[j] }) + sort.Slice(st, func(i, j int) bool { return st[i] < st[j] }) + over := 0 + for _, d := range bt { + if d > c.interval { + over++ + } + } + var height uint64 + hs := sumHeap(stores) + holes, live, scanned, moved, syncs, syncBytes, heapFsync, deltaSync := hs.holes, hs.live, hs.scanned, hs.moved, hs.syncs, hs.bytes, hs.fsync, hs.delta + var permMerges, permFolds, permPacks, permIndexBytes uint64 + for _, s := range stores { + if s.height > height { + height = s.height + } + for _, sh := range s.kv.Shards { + if sh.Perm != nil { + m, f, pk, ib := sh.Perm.Counters() + permMerges, permFolds, permPacks, permIndexBytes = permMerges+m, permFolds+f, permPacks+pk, permIndexBytes+ib + } + } + } + b, _ := json.Marshal(map[string]any{ + "elapsedSec": int(time.Since(start).Seconds()), "blocks": t.blocks.Load(), "height": height, + "last10s": map[string]any{"blocks": len(bt), "over": over, + "blockP50ms": float64(pct(bt, .5)) / 1e6, "blockP90ms": float64(pct(bt, .9)) / 1e6, "blockMaxMs": float64(pct(bt, 1)) / 1e6, + "sealP50ms": float64(pct(st, .5)) / 1e6, "sealP90ms": float64(pct(st, .9)) / 1e6, "sealMaxMs": float64(pct(st, 1)) / 1e6, + "storeSealP90ms": storeP90}, + "maintenanceInFlight": t.inFlight.Load(), "mismatches": t.mismatches.Load(), + "heapHoleMB": float64(holes) / 1e6, "heapLiveMB": float64(live) / 1e6, + "heapScannedMB": float64(scanned) / 1e6, "heapMovedMB": float64(moved) / 1e6, + "heapSyncs": syncs, "heapSyncKBAvg": float64(syncBytes) / 1e3 / float64(max(syncs, 1)), + "permMerges": permMerges, "permFolds": permFolds, "permPacks": permPacks, "permIndexMB": float64(permIndexBytes) / 1e6, + "heapFsyncMsAvg": float64(heapFsync) / 1e6 / float64(max(syncs, 1)), "heapDeltaMsAvg": float64(deltaSync) / 1e6 / float64(max(syncs, 1)), + "heapReleases": hs.releases, "heapReleaseMs": hs.release.Milliseconds(), "heapSnapshots": hs.snapshots, "heapSnapshotMs": hs.snapshot.Milliseconds(), + }) + return b +} + +// heapSplit sums the heap layer's counters across the stores: the +// mover's and the sync's cost, cumulative since the start. +type heapSplit struct { + holes, live int64 + scanned, moved, syncs, bytes uint64 + fsync, delta time.Duration + releases, snapshots uint64 + release, snapshot time.Duration +} + +func (a heapSplit) minus(b heapSplit) heapSplit { + return heapSplit{holes: a.holes, live: a.live, + scanned: a.scanned - b.scanned, moved: a.moved - b.moved, syncs: a.syncs - b.syncs, bytes: a.bytes - b.bytes, + fsync: a.fsync - b.fsync, delta: a.delta - b.delta, releases: a.releases - b.releases, snapshots: a.snapshots - b.snapshots, + release: a.release - b.release, snapshot: a.snapshot - b.snapshot} +} + +func sumHeap(stores []*store) (hs heapSplit) { + for _, s := range stores { + for _, sh := range s.kv.Shards { + if sh.Heap == nil { + continue + } + h, l := sh.Heap.HoleRatio() + hs.holes, hs.live = hs.holes+h, hs.live+l + sc, mv := sh.Heap.Cleaned() + hs.scanned, hs.moved = hs.scanned+sc, hs.moved+mv + n, b, hf, ds := sh.Heap.SyncCost() + hs.syncs, hs.bytes, hs.fsync, hs.delta = hs.syncs+n, hs.bytes+b, hs.fsync+hf, hs.delta+ds + r, rt, sn, st := sh.Heap.MoverCost() + hs.releases, hs.release, hs.snapshots, hs.snapshot = hs.releases+r, hs.release+rt, hs.snapshots+sn, hs.snapshot+st + } + } + return hs +} + +// permValue is the value a permanent key carries: derived from the +// key, so that any read of the key can be checked without remembering +// what was written. Its length spreads over [min, max] by the key. +func permValue(k [32]byte, min, max uint) []byte { + n := int(min) + if max > min { + n += int(k[31]) * int(max-min) / 255 + } + v := make([]byte, n) + for i := 0; i < n; i += len(k) { + copy(v[i:], k[:]) + } + return v +} + func fail(what string, err error) { fmt.Fprintln(os.Stderr, "bdbench:", what+":", err) os.Exit(1) @@ -406,7 +602,7 @@ func main() { "dir": c.dir, "stores": c.stores, "duration": c.duration.String(), "interval": c.interval.String(), "shards": c.shards, "sealLimit": c.sealLimit, "window": c.window, "compressEvery": c.compressEvery, "packEvery": c.packEvery, "dynaPuts": c.dynaPuts, "permPuts": c.permPuts, "reads": c.reads, "hotKeys": c.hotKeys, - "valueMin": c.valueMin, "valueMax": c.valueMax, "seed": c.seed, "started": time.Now().UTC().Format(time.RFC3339), + "valueMin": c.valueMin, "valueMax": c.valueMax, "seed": c.seed, "dynaHeap": c.dynaHeap || c.permFiles, "permFiles": c.permFiles, "maintenancePhase": c.phase, "started": time.Now().UTC().Format(time.RFC3339), }, "", " ") if err := os.WriteFile(filepath.Join(c.dir, "run.json"), runJSON, 0o644); err != nil { fail("run.json", err) @@ -439,7 +635,9 @@ func main() { _ = csvw.Write([]string{"minute", "blocks", "over_budget", "block_p50_ms", "block_p90_ms", "block_max_ms", "seal_p50_ms", "seal_p90_ms", "seal_max_ms", "dyna_put_p99_us", "perm_put_p99_us", "read_p99_us", "compress_passes", "compress_s", "merge_passes", "merge_s", "pack_passes", "pack_s", "skipped", - "read_MBps", "write_MBps", "store_MB", "files", "perm_history", "dyna_history", "bloom_MB", "mismatches"}) + "read_MBps", "write_MBps", "store_MB", "files", "perm_history", "dyna_history", "bloom_MB", "mismatches", + "heap_syncs", "heap_fsync_ms_avg", "heap_sync_MB_avg", "heap_moved_MB", "heap_hole_MB", "heap_live_MB", + "heap_releases", "heap_release_ms", "heap_snapshots", "heap_snapshot_ms"}) csvw.Flush() fmt.Printf("bdbench: %d store(s) x %d shards, seal limit %d, window %d, maintenance every %d blocks, pack every %d; per block per store %d dyna + %d perm puts, %d reads; %s blocks for %s; %s\n", @@ -450,6 +648,7 @@ func main() { deadline := start.Add(c.duration) ioR0, ioW0 := procIO() period := start + var heap0 heapSplit report := func() { elapsed := time.Since(period) @@ -483,11 +682,20 @@ func main() { } minute := int(time.Since(start) / time.Minute) mism := t.mismatches.Load() - fmt.Printf("%3dm blocks %4d | block p50/p90/max %s/%s/%s ms | seal p50/p90/max %s/%s/%s ms | put p99 dyna %s perm %s us | read p99 %s us | maint compress %d (%.1fs) merge %d (%.1fs) pack %d (%.1fs) skipped %d | disk r %.0f w %.0f MB/s | store %.0f MB %d files | history perm %d dyna %d | bloom %.0f MB | mismatches %d%s\n", + heap1 := sumHeap(stores) + hs := heap1.minus(heap0) + heap0 = heap1 + heapNote := "" + if hs.syncs > 0 { + heapNote = fmt.Sprintf(" | heap fsync %.1f ms x %d (%.1f MB) moved %.0f MB hole %.0f MB live %.0f MB releases %d (%d ms) snapshots %d (%d ms)", + float64(hs.fsync)/1e6/float64(hs.syncs), hs.syncs, float64(hs.bytes)/1e6/float64(hs.syncs), float64(hs.moved)/1e6, float64(hs.holes)/1e6, float64(hs.live)/1e6, + hs.releases, hs.release.Milliseconds(), hs.snapshots, hs.snapshot.Milliseconds()) + } + fmt.Printf("%3dm blocks %4d | block p50/p90/max %s/%s/%s ms | seal p50/p90/max %s/%s/%s ms | put p99 dyna %s perm %s us | read p99 %s us | maint compress %d (%.1fs) merge %d (%.1fs) pack %d (%.1fs) skipped %d | disk r %.0f w %.0f MB/s | store %.0f MB %d files | history perm %d dyna %d | bloom %.0f MB | mismatches %d%s%s\n", minute, len(bt), ms(pct(bt, .5)), ms(pct(bt, .9)), ms(pct(bt, 1)), ms(pct(st, .5)), ms(pct(st, .9)), ms(pct(st, 1)), us(pct(dp, .99)), us(pct(pp, .99)), us(pct(rt, .99)), passes["compress"], spent["compress"].Seconds(), passes["merge"], spent["merge"].Seconds(), passes["pack"], spent["pack"].Seconds(), skipped, - rMB, wMB, float64(bytes)/1e6, files, permHist, dynaHist, float64(bloom)/1e6, mism, note) + rMB, wMB, float64(bytes)/1e6, files, permHist, dynaHist, float64(bloom)/1e6, mism, heapNote, note) _ = csvw.Write([]string{strconv.Itoa(minute), strconv.Itoa(len(bt)), strconv.FormatUint(over, 10), ms(pct(bt, .5)), ms(pct(bt, .9)), ms(pct(bt, 1)), ms(pct(st, .5)), ms(pct(st, .9)), ms(pct(st, 1)), us(pct(dp, .99)), us(pct(pp, .99)), us(pct(rt, .99)), @@ -497,10 +705,31 @@ func main() { strconv.FormatFloat(rMB, 'f', 1, 64), strconv.FormatFloat(wMB, 'f', 1, 64), strconv.FormatFloat(float64(bytes)/1e6, 'f', 0, 64), strconv.Itoa(files), strconv.Itoa(permHist), strconv.Itoa(dynaHist), strconv.FormatFloat(float64(bloom)/1e6, 'f', 1, 64), - strconv.FormatUint(mism, 10)}) + strconv.FormatUint(mism, 10), + strconv.FormatUint(hs.syncs, 10), strconv.FormatFloat(float64(hs.fsync)/1e6/float64(max(hs.syncs, 1)), 'f', 1, 64), + strconv.FormatFloat(float64(hs.bytes)/1e6/float64(max(hs.syncs, 1)), 'f', 1, 64), strconv.FormatFloat(float64(hs.moved)/1e6, 'f', 0, 64), + strconv.FormatFloat(float64(hs.holes)/1e6, 'f', 0, 64), strconv.FormatFloat(float64(hs.live)/1e6, 'f', 0, 64), + strconv.FormatUint(hs.releases, 10), strconv.FormatInt(hs.release.Milliseconds(), 10), + strconv.FormatUint(hs.snapshots, 10), strconv.FormatInt(hs.snapshot.Milliseconds(), 10)}) csvw.Flush() } + // The live state, every two seconds, beside the per-minute rows + liveStop := make(chan struct{}) + go func() { + tick := time.NewTicker(2 * time.Second) + defer tick.Stop() + for { + select { + case <-tick.C: + _ = os.WriteFile(filepath.Join(c.dir, "live.json"), t.liveState(c, stores, start), 0o644) + case <-liveStop: + return + } + } + }() + defer close(liveStop) + // Every store drives its own blocks; the first error stops the run. stop := make(chan struct{}) var stopOnce sync.Once @@ -551,6 +780,39 @@ wait: fail("close", err) } } + // Every store is reopened and read back: every sampled permanent + // key with its derived value, every checked hot key with its last + // value. What a seal made durable must be there after a close + // and an open, and a run that loses one fails. + var wrong int + for _, s := range stores { + re, err := blockchainDB.OpenKVShard(s.kv.Directory) + if err != nil { + fail(fmt.Sprintf("store %d reopen", s.id), err) + } + bad := 0 + for i, k := range s.permKeys { + if v, err := re.GetDeep(k); err != nil || !bytes.Equal(v, permValue(k, c.valueMin, c.valueMax)) { + if bad < 3 { + fmt.Printf(" sampled key %d of %d: err=%v got %d bytes want %d\n", i, len(s.permKeys), err, len(v), len(permValue(k, c.valueMin, c.valueMax))) + } + bad++ + } + } + for k, want := range s.last { + if v, err := re.GetDyna(k); err != nil || !bytes.Equal(v, want) { + bad++ + } + } + fmt.Printf("reopen store %d: %d permanent + %d dynamic keys read back, %d wrong\n", s.id, len(s.permKeys), len(s.last), bad) + wrong += bad + if err := re.Close(); err != nil { + fail("close after reopen", err) + } + } + if wrong > 0 { + fail("reopen", fmt.Errorf("%d keys lost or wrong after close and reopen", wrong)) + } if runErr != nil { fail("run", runErr) } diff --git a/cmd/bdbench/tools/README.md b/cmd/bdbench/tools/README.md new file mode 100644 index 0000000..3acbfef --- /dev/null +++ b/cmd/bdbench/tools/README.md @@ -0,0 +1,36 @@ +# bdbench tools + +Samplers that run beside a bdbench run and write one line every two +seconds, so that a minute's row can be taken apart afterwards. + +- `timeline.sh ` samples the run's `live.json` into + `.timeline`: the last ten seconds' seal p50/p90/max, the heap's + cumulative fsync average and bytes per sync, snapshots, releases, + moved bytes, hole and live bytes, perm merges and folds, and each + store's seal p90 (`ps`). +- `sys-timeline.sh ` samples the machine into + `.sys`: dirty and free memory, the NVMe device's write count, + write ticks and in-flight requests from `/proc/diskstats`, and the + load average. + +Both wait for `//live.json` to appear and stop when it +goes away (the run directory is removed). Start them detached before +the run: + + setsid nohup tools/timeline.sh $S heap-2a >/dev/null 2>&1 & + setsid nohup tools/sys-timeline.sh $S heap-2a >/dev/null 2>&1 & + +## Reading them + +Difference the cumulative fields between samples. The device's write +ticks per 2-second window is the queue depth times the latency: 1-3 s +on this disk when healthy, 100-500 s when the drive is stalling. + +A disk stall and a store tail look alike in the per-minute row (seal +p50 unchanged, p90 200-500 ms) and different here: a stall raises the +heap's fsync average for every store at once, the device's write ticks +with it, and the store's own counters (moved, snapshots, releases) show +no step. A store tail lines up with a step in one of those counters +and the device stays quiet. The disk this was learned on, and how it +was fixed, are in the operator's machine notes rather than here: on this +machine `~/infrastructure/disk-trim.md`. diff --git a/cmd/bdbench/tools/sys-timeline.sh b/cmd/bdbench/tools/sys-timeline.sh new file mode 100755 index 0000000..646951f --- /dev/null +++ b/cmd/bdbench/tools/sys-timeline.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# System counters every 2 s while a run's live.json exists: memory and the device +S=$1; run=$2 +until [ -f $S/$run/live.json ]; do sleep 2; done +while [ -f $S/$run/live.json ]; do + t=$(jq -r .elapsedSec $S/$run/live.json 2>/dev/null) + m=$(awk '/^(Dirty|Writeback|MemFree|MemAvailable|Cached):/{printf "%s=%d ", $1, $2/1024}' /proc/meminfo) + d=$(grep -w nvme0n1 /proc/diskstats | awk '{printf "wr=%d wr_ms=%d inflight=%d io_ms=%d", $8, $11, $12, $13}') + l=$(cut -d' ' -f1 /proc/loadavg) + echo "t=$t $m $d load=$l" >> $S/$run.sys + sleep 2 +done diff --git a/cmd/bdbench/tools/timeline.sh b/cmd/bdbench/tools/timeline.sh new file mode 100755 index 0000000..11ba052 --- /dev/null +++ b/cmd/bdbench/tools/timeline.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Samples a run's live.json every 2 s into .timeline (one JSON line each) +S=$1; run=$2 +until [ -f $S/$run/live.json ]; do sleep 2; done +while [ -f $S/$run/live.json ]; do + jq -c '{t:.elapsedSec, s50:.last10s.sealP50ms, s90:.last10s.sealP90ms, smax:.last10s.sealMaxMs, fs:.heapFsyncMsAvg, kb:.heapSyncKBAvg, syncs:.heapSyncs, snaps:.heapSnapshots, snapms:.heapSnapshotMs, rel:.heapReleases, relms:.heapReleaseMs, mv:.heapMovedMB, hole:.heapHoleMB, live:.heapLiveMB, merges:.permMerges, folds:.permFolds, inflight:.maintenanceInFlight, ps:((.last10s.storeSealP90ms // [])|map(floor))}' $S/$run/live.json 2>/dev/null >> $S/$run.timeline + sleep 2 +done diff --git a/database/heap.go b/database/heap.go new file mode 100644 index 0000000..166ac19 --- /dev/null +++ b/database/heap.go @@ -0,0 +1,1577 @@ +package blockchainDB + +import ( + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +// HeapStore is the dynamic layer as files of entries and a map of keys +// (proposal docs/proposals/2026-09-16-entries-written-once.md): an +// entry is written once at the end of a file, a key is (file, offset, +// length), and space comes back by deleting a file once nothing in it +// is live -- never by rewriting an index or a filter. +// +// Files, in one directory: +// +// heap-N.dat entries, [len u32][height u64][key 32][value][crc32 +// of height+key+value], 8-byte aligned, appended in the +// order written. A block appends to the current data +// file; the mover appends to a file of its own, so the +// two never share a barrier. A file is rolled at +// HeapFileBytes and deleted once nothing in it is live. +// index-G.log generation G of the key map: a snapshot record of the +// whole map and the point in the data files from which +// the deltas after it are replayed. A snapshot starts a +// new generation in a file of its own, switched to by +// rename. +// +// A block's delta is appended to the block's data file right behind +// the block's entries, as an entry under a reserved key, so a block is +// ONE fsync per shard (entries and delta together) rather than two +// (measured: the second barrier was a third of the sync's cost, and +// the barrier count is what the device queue charges for). The delta +// names no key: a block's entries are contiguous in its file and each +// carries its key, so the delta is the RANGES the block wrote -- its +// own appends, and the mover's copies named this block -- and the few +// copies that arrived dead, a few dozen bytes a block (a delta of +// 44-byte records was a fifth of the heap's writes, all of it dead at +// the next snapshot and churned by the mover). Replay scans the +// ranges in order, later naming winning, and the last delta is +// trusted only if every entry in its ranges checks; an earlier delta +// was followed by a later block's fsync, which covers it. +// +// The key map is in memory for the live key set (spec 1.2: memory that +// scales with the working set). Open loads the newest whole +// generation's snapshot, replays the deltas in the data files after +// its replay point, and derives every file's live and dead bytes; a +// data file nothing names is deleted then, and one that is missing +// while named is an error. +// +// Durability (spec 1.8). The block sync fsyncs the data files the +// block wrote and then appends and fsyncs the block's delta, so an +// index entry is durable only after the entry it names is. A slot the +// last durable index names is never overwritten: a key rewritten in a +// later block takes a new slot, and its old slot is dead where it lies +// until its file is deleted -- and a file is deleted only after the +// delta naming the mover's copies out of it is durable, one sync late, +// the way spec 2.6 defers deletion (never unlink what a durable index +// names). A key rewritten again within the same block reuses the +// slot it took this block, since nothing durable names it yet. A +// crash therefore leaves every durable entry intact and every torn +// slot unnamed; the checksum catches a torn slot a stale index could +// name; and RepairHeapStore rebuilds the map from the data alone, +// since every entry carries its key and the height that wrote it. +type HeapStore struct { + Directory string + + mu sync.RWMutex + files map[uint32]*heapFile + cur *heapFile // The file the block appends to + mov *heapFile // The file the mover appends to + nextID uint32 + index map[[32]byte]slot + height uint64 // The block being written; slots taken in it may be rewritten in place + + // The block's delta in the making: the ranges the block appended + // (the mover's named copies first, then the block's own) and the + // copies that arrived dead; dirty the files written since the last + // sync; release the files the mover emptied, deleted once the + // delta naming their copies is durable. + ranges []heapRange + excluded []heapRange + dirty map[uint32]*heapFile + release []uint32 + // deltaAt is where the last delta ended: the replay point a + // snapshot records, and what a file is cut back to on open + deltaAt struct { + file uint32 + off int64 + } + + // syncMu serializes block syncs with each other and with a + // snapshot, so the map a snapshot writes is exactly the state of + // the last finished delta and no delta is in flight into a log + // about to be retired. + syncMu sync.Mutex + log *os.File + gen uint64 + snapAt uint64 // The block the key map was last snapshotted at + snapPhase uint64 // Blocks this shard's snapshot cadence is offset by + + closed bool + liveBytes int64 + deadBytes int64 + bound bool // The size bound engaged: the mover takes files below the ratio + + putTotal, putInPlace, putAppend atomic.Uint64 + lookups, hits atomic.Uint64 + cleanedBytes, movedBytes atomic.Uint64 + // The sync's cost, split: nanoseconds in the data files' fsyncs + // and in the delta's write and fsync, and the syncs and bytes + syncs, syncHeapNs, syncLogNs, syncBytes atomic.Uint64 + // The mover's cost on and off the block's path: files unlinked by + // a block's finish and the time in those unlinks; snapshots + // written and the time in them + releases, releaseNs, snapshotsN, snapshotNs atomic.Uint64 + unlink []uint32 // Released files not yet deleted +} + +// heapFile is one data file and its accounting. +type heapFile struct { + id uint32 + f *os.File + size int64 + live, dead int64 + deltas int64 // Bytes of delta entries not yet superseded by a snapshot: live until then + firstBlock uint64 // The block that first appended to it + cleaning bool // Taken by the pass in progress + inflight int // Copies reserved in it and not yet written: its size runs ahead of its bytes + releasing bool // Emptied by a pass; deleted by the next sync + // Bytes of ranges in it that the deltas of the current generation + // name, and where the last of them ends. A durable delta names + // the file, so the file stays until a snapshot supersedes the + // delta, whatever is live in it (spec 1.7): never unlink what a + // durable index names. + named, namedEnd int64 +} + +// nameRanges records that a delta of the current generation names +// these ranges. The caller holds the lock. +func (h *HeapStore) nameRanges(ranges []heapRange) { + for _, r := range ranges { + hf := h.files[r.file] + if hf == nil || r.to <= r.from { + continue + } + hf.named += int64(r.to - r.from) + if int64(r.to) > hf.namedEnd { + hf.namedEnd = int64(r.to) + } + } +} + +// heapRange is a stretch of one data file: what a delta names. +type heapRange struct { + file uint32 + from, to uint32 +} + +// slot is where an entry lives: its file, its offset there, the value +// length, and the block that took it, which decides whether a rewrite +// may reuse it in place. +type slot struct { + file uint32 + off uint32 + n uint32 + block uint64 +} + +const ( + heapHeader = 4 + 8 + 32 // len, height, key + heapTrailer = 4 // crc32 of height+key+value + heapAlign = 8 // Entries start on an 8-byte boundary + heapMagic = 0x48454150 // "HEAP", an index record's marker + heapSnapshot = 0x50414E53 // "SNAP", the record that starts a generation + heapIndexHdr = 4 + 8 + 4 + 4 + 4 // magic, height, count, replay file, replay offset + heapIndexRec = 32 + 4 + 4 + 4 +) + +// heapDeltaKey is the reserved key a delta entry is written under in +// a data file. A real key is a hash; the odds of one being all ones +// are those of a hash collision. +var heapDeltaKey = [32]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + +// HeapFileBytes is the size a data file is rolled at, and +// HeapFileBlocks the age: a shard that appends little per block (many +// shards, or a quiet one) would otherwise keep one file current for +// long stretches, and the current file is never the mover's, so its +// dead bytes could not be reclaimed (measured at 64 shards: 34 KB a +// block, no file rolled in three minutes, the store growing 2.3 GB a +// minute with the mover idle). +var HeapFileBytes int64 = 16 << 20 + +var HeapFileBlocks uint64 = 64 + +// HeapCleanBytes bounds one mover pass by the bytes it copies. The +// pass syncs its own copies, so the bound is not about a block's +// barrier but about the device queue the barrier shares: 16 MB +// passes put the seal's p90 at 150-250 ms in the minutes they ran +// (run 7), 4 MB releases more than the soak appends per shard per +// cadence (~4 MB, of which a quarter to a third is live). +var HeapCleanBytes int64 = 4 << 20 + +// HeapStoreCleanBytes is a store's mover budget per pass, shared out +// among its shards: a store sharded eight ways moves the same bytes a +// pass as one sharded once. Measured: with one shard and the +// per-shard budget alone the store grew to 6 GB in five minutes +// where eight shards held it under 4. +var HeapStoreCleanBytes int64 = 32 << 20 + +// HeapCleanFiles and HeapScanBytes bound a pass by the files it takes +// and the bytes it reads: with hot keys most of a file is dead and +// costs nothing to copy, so what limits the mover's pace is how much +// it looks at. At the soak's rate a shard makes ~50 MB of dead bytes +// per cadence; a pass that scans 128 MB releases more than that with +// room to spare (four 16 MB files did not, and the store floated with +// the size bound engaged). +var ( + HeapCleanFiles = 8 + HeapScanBytes int64 = 128 << 20 +) + +// HeapCleanRatio is the dead fraction a file must reach before the +// mover takes it -- unless the heap is over its size bound, when the +// deadest file is taken at HeapCleanFloor or more. The bound has +// hysteresis: it engages when dead bytes exceed live by +// HeapBoundOn and releases when they fall below live again, so the +// movers of nine stores in lockstep do not all engage on the same +// block and then all disengage (measured: p90 461-704 ms in the +// minutes the bound flipped, 70-90 ms otherwise). +var ( + HeapCleanRatio = 0.5 + HeapCleanFloor = 0.25 + HeapBoundOn = 1.5 +) + +// HeapSnapshotBlocks is how many blocks pass between key-map +// snapshots; between them the generation's deltas are what open +// replays. Counted in blocks, not maintenance calls, so the cadence +// of the caller does not set the cadence of the snapshots. +var HeapSnapshotBlocks uint64 = 100 + +// HeapSnapshotPinnedFiles is how many files' worth of bytes may be +// held only by their deltas before a snapshot is taken early, +// whatever the block count. +var HeapSnapshotPinnedFiles int64 = 2 + +// HeapCleanPeriod is the blocks over which a store spends +// HeapStoreCleanBytes: the store's mover rate is the one divided by +// the other, and a maintenance call moves what the blocks since the +// last call earned (KVShard.Compress). +var HeapCleanPeriod uint64 = 20 + +// entrySize is the bytes an entry of n value bytes takes, aligned. +func entrySize(n int) int64 { + return (int64(heapHeader+n+heapTrailer) + heapAlign - 1) &^ (heapAlign - 1) +} + +func dataName(id uint32) string { return fmt.Sprintf("heap-%06d.dat", id) } +func indexName(gen uint64) string { return fmt.Sprintf("index-%06d.log", gen) } + +// NewHeapStore creates an empty heap in directory, replacing anything +// there. +func NewHeapStore(directory string) (*HeapStore, error) { + os.RemoveAll(directory) + if err := os.MkdirAll(directory, 0o755); err != nil { + return nil, err + } + h := &HeapStore{Directory: directory} + return h, h.Open() +} + +// OpenHeapStore opens the heap in directory as it was left. A heap +// with data files but no index generation is refused: that is what +// RepairHeapStore is for, and it needs the committed height, which +// only the store above knows. +func OpenHeapStore(directory string) (*HeapStore, error) { + dataIDs, gens, err := listHeap(directory) + if err != nil { + return nil, fmt.Errorf("open heap at %s: %w", directory, err) + } + if len(dataIDs) > 0 && len(gens) == 0 { + return nil, fmt.Errorf("open heap at %s: %w", directory, ErrHeapNeedsRepair) + } + h := &HeapStore{Directory: directory} + return h, h.Open() +} + +// ErrHeapNeedsRepair says the heap's data is there and its index is +// not: RepairHeapStore rebuilds the index from the data. +var ErrHeapNeedsRepair = errors.New("heap has data but no index; repair it") + +// listHeap names the data files and the whole index generations in a +// directory, in order. A generation still being written (.tmp) is +// not whole and is removed. +func listHeap(directory string) (dataIDs []uint32, gens []uint64, err error) { + entries, err := os.ReadDir(directory) + if err != nil { + return nil, nil, err + } + for _, e := range entries { + name := e.Name() + switch { + case strings.HasPrefix(name, "heap-") && strings.HasSuffix(name, ".dat"): + id, err := strconv.ParseUint(strings.TrimSuffix(strings.TrimPrefix(name, "heap-"), ".dat"), 10, 32) + if err == nil { + dataIDs = append(dataIDs, uint32(id)) + } + case strings.HasPrefix(name, "index-") && strings.HasSuffix(name, ".log"): + g, err := strconv.ParseUint(strings.TrimSuffix(strings.TrimPrefix(name, "index-"), ".log"), 10, 64) + if err == nil { + gens = append(gens, g) + } + case strings.HasSuffix(name, segTmpSuffix): + os.Remove(filepath.Join(directory, name)) + } + } + sort.Slice(dataIDs, func(i, j int) bool { return dataIDs[i] < dataIDs[j] }) + sort.Slice(gens, func(i, j int) bool { return gens[i] < gens[j] }) + return dataIDs, gens, nil +} + +// Open loads the key map from the newest generation, opens the data +// files and derives their accounting. Idempotent. +func (h *HeapStore) Open() (err error) { + h.mu.Lock() + defer h.mu.Unlock() + if h.files != nil { + return nil + } + dataIDs, gens, err := listHeap(h.Directory) + if err != nil { + return err + } + h.index = map[[32]byte]slot{} + h.dirty = map[uint32]*heapFile{} + h.files = map[uint32]*heapFile{} + h.release, h.ranges, h.excluded = nil, nil, nil + h.closed = false + h.liveBytes, h.deadBytes = 0, 0 + // The newest whole generation is the index; older ones are what a + // snapshot left behind when it could not delete them + if len(gens) > 0 { + h.gen = gens[len(gens)-1] + for _, g := range gens[:len(gens)-1] { + os.Remove(filepath.Join(h.Directory, indexName(g))) + } + if err = h.replayGeneration(); err != nil { + return err + } + } else { + h.gen = 1 + // A fresh store: the first generation, written with the lock + // released since startGeneration takes it itself + h.mu.Unlock() + err = h.startGeneration() + h.mu.Lock() + if err != nil { + return err + } + } + for _, id := range dataIDs { + if id >= h.nextID { + h.nextID = id + 1 + } + } + if err = h.openFiles(dataIDs); err != nil { + return err + } + if err = h.replayDeltas(dataIDs); err != nil { + return err + } + return h.deriveFiles(dataIDs) +} + +// openFiles opens every data file present. The caller holds the lock. +func (h *HeapStore) openFiles(dataIDs []uint32) error { + for _, id := range dataIDs { + f, err := os.OpenFile(filepath.Join(h.Directory, dataName(id)), os.O_RDWR, 0o644) + if err != nil { + return err + } + st, err := f.Stat() + if err != nil { + f.Close() + return err + } + h.files[id] = &heapFile{id: id, f: f, size: st.Size()} + } + return nil +} + +// replayGeneration loads the current generation's snapshot; the +// deltas after it are in the data files and replayDeltas applies +// them. The caller holds the lock. +func (h *HeapStore) replayGeneration() (err error) { + path := filepath.Join(h.Directory, indexName(h.gen)) + if h.log, err = os.OpenFile(path, os.O_RDWR|os.O_APPEND, 0o644); err != nil { + return err + } + buf, err := os.ReadFile(path) + if err != nil { + return err + } + n, err := h.applyRecord(buf) + if err != nil { + return fmt.Errorf("heap: generation %d: %w", h.gen, err) + } + if n != len(buf) { + return fmt.Errorf("heap: generation %d has %d trailing bytes", h.gen, len(buf)-n) + } + return nil +} + +// replayDeltas applies every delta entry in the data files after the +// snapshot's replay point, in file and offset order. The last delta +// is trusted only if every entry it names checks: one fsync covered +// it and its entries together, and a crash inside that fsync can +// leave the delta durable and an entry torn. An earlier delta was +// followed by a later block's fsync. The caller holds the lock; the +// files are open. +func (h *HeapStore) replayDeltas(dataIDs []uint32) error { + type found struct { + file uint32 + off int64 + end int64 + recs []byte + } + var deltas []found + for _, id := range dataIDs { + if id < h.deltaAt.file { + continue + } + hf := h.files[id] + if hf == nil { + continue + } + buf := make([]byte, hf.size) + if _, err := hf.f.ReadAt(buf, 0); err != nil && !errors.Is(err, io.EOF) { + return err + } + at := int64(0) + if id == h.deltaAt.file { + at = h.deltaAt.off + } + for at < int64(len(buf)) { + size, _, key, value, ok := decodeEntry(buf[at:]) + if size == 0 { + break + } + if ok && key == heapDeltaKey { + deltas = append(deltas, found{file: id, off: at, end: at + size, recs: append([]byte(nil), value...)}) + } + at += size // A damaged entry is stepped over; if it is named, Get reports it + } + } + for i, d := range deltas { + if !h.applyDelta(d.recs, i == len(deltas)-1) { + break // Torn, or the last block's entries do not all check: it did not commit + } + h.deltaAt.file, h.deltaAt.off = d.file, d.end + h.files[d.file].deltas += d.end - d.off + } + return nil +} + +// startGeneration begins an index generation: a snapshot of the map +// with the replay point -- where the last delta ended -- written +// aside and fsynced, renamed into place, the directory fsynced, and +// the previous generation's file removed once the new one is durable. +// The map and the point are taken together under the lock; a delta +// the seal appends after that lies past the point and is replayed +// over the snapshot on open, which is idempotent. The snapshot's +// write holds no lock (measured: a 3 MB snapshot under syncMu put +// the seal's p90 at 276 ms every hundredth block). A delta's bytes +// before the point are dead once the generation is durable. +func (h *HeapStore) startGeneration() error { + next := h.gen + if h.log != nil { + next = h.gen + 1 // A snapshot starts the generation after the current one + } + path := filepath.Join(h.Directory, indexName(next)) + tmp := path + segTmpSuffix + h.mu.Lock() + all := func(emit func(key [32]byte)) { + for key := range h.index { + emit(key) + } + } + snap := h.encodeIndexOf(heapSnapshot, all, len(h.index)) + superseded := map[uint32]int64{} + namedThen := map[uint32]int64{} + for id, hf := range h.files { + superseded[id] = hf.deltas + namedThen[id] = hf.named + } + h.mu.Unlock() + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return err + } + if _, err = f.Write(snap); err != nil { + f.Close() + return err + } + if err = fsync(f); err != nil { + f.Close() + return err + } + if err = f.Close(); err != nil { + return err + } + if err = os.Rename(tmp, path); err != nil { + return err + } + if err = fsyncDir(h.Directory); err != nil { + return err + } + h.mu.Lock() + defer h.mu.Unlock() + old := h.log + if h.log, err = os.OpenFile(path, os.O_RDWR|os.O_APPEND, 0o644); err != nil { + return err + } + if old != nil { + old.Close() + os.Remove(filepath.Join(h.Directory, indexName(h.gen))) + } + h.gen = next + // The deltas the snapshot covers are dead where they lie + for id, n := range superseded { + if hf := h.files[id]; hf != nil && n > 0 { + hf.deltas -= n + hf.live -= n + hf.dead += n + h.liveBytes -= n + h.deadBytes += n + } + } + // The ranges those deltas named are no longer named by anything + // durable but the snapshot's own slots + for id, n := range namedThen { + if hf := h.files[id]; hf != nil { + hf.named -= n + if hf.named == 0 { + hf.namedEnd = 0 + } + } + } + return nil +} + +// fsyncDir makes a directory's entries durable: a rename or an unlink +// is not on disk until the directory is. +func fsyncDir(directory string) error { + d, err := os.Open(directory) + if err != nil { + return err + } + defer d.Close() + return fsync(d) +} + +// encodeDelta is the block's delta: marker, height, the ranges the +// block wrote (the mover's named copies first, then the block's own +// appends, so a later write of a key wins on replay) and the copies +// that arrived dead, checksummed. The caller holds the lock. +func (h *HeapStore) encodeDelta() []byte { + const rangeRec = 4 + 4 + 4 + buf := make([]byte, 4+8+4+4+len(h.ranges)*rangeRec+len(h.excluded)*rangeRec+4) + binary.LittleEndian.PutUint32(buf, heapMagic) + binary.LittleEndian.PutUint64(buf[4:], h.height) + binary.LittleEndian.PutUint32(buf[12:], uint32(len(h.ranges))) + binary.LittleEndian.PutUint32(buf[16:], uint32(len(h.excluded))) + at := 20 + for _, r := range append(append([]heapRange(nil), h.ranges...), h.excluded...) { + binary.LittleEndian.PutUint32(buf[at:], r.file) + binary.LittleEndian.PutUint32(buf[at+4:], r.from) + binary.LittleEndian.PutUint32(buf[at+8:], r.to) + at += rangeRec + } + binary.LittleEndian.PutUint32(buf[at:], crc32.ChecksumIEEE(buf[:at])) + return buf +} + +// decodeDelta reads a delta's ranges and exclusions. +func decodeDelta(buf []byte) (height uint64, ranges, excluded []heapRange, ok bool) { + const rangeRec = 4 + 4 + 4 + if len(buf) < 20 || binary.LittleEndian.Uint32(buf) != heapMagic { + return + } + nr, ne := int(binary.LittleEndian.Uint32(buf[12:])), int(binary.LittleEndian.Uint32(buf[16:])) + end := 20 + (nr+ne)*rangeRec + if len(buf) < end+4 || crc32.ChecksumIEEE(buf[:end]) != binary.LittleEndian.Uint32(buf[end:]) { + return + } + height = binary.LittleEndian.Uint64(buf[4:]) + at := 20 + for i := 0; i < nr+ne; i++ { + r := heapRange{file: binary.LittleEndian.Uint32(buf[at:]), from: binary.LittleEndian.Uint32(buf[at+4:]), to: binary.LittleEndian.Uint32(buf[at+8:])} + if i < nr { + ranges = append(ranges, r) + } else { + excluded = append(excluded, r) + } + at += rangeRec + } + return height, ranges, excluded, true +} + +// applyDelta names every entry in the delta's ranges, in order, later +// naming winning, except the excluded copies; verify makes it refuse +// a delta any of whose entries does not check. The caller holds the +// lock; the files are open. +func (h *HeapStore) applyDelta(recs []byte, verify bool) bool { + height, ranges, excluded, ok := decodeDelta(recs) + if !ok { + return false + } + skip := map[heapRange]bool{} + for _, e := range excluded { + skip[heapRange{file: e.file, from: e.from}] = true + } + type naming struct { + key [32]byte + s slot + } + var names []naming + for _, r := range ranges { + hf := h.files[r.file] + if hf == nil || int64(r.to) > hf.size { + return false + } + buf := make([]byte, r.to-r.from) + if _, err := hf.f.ReadAt(buf, int64(r.from)); err != nil { + return false + } + at := int64(0) + for at < int64(len(buf)) { + size, _, key, value, ok := decodeEntry(buf[at:]) + if size == 0 || !ok && verify { + return false // Torn; or the last block's entries do not all check + } + off := int64(r.from) + at + // A damaged entry of a committed block is named all the + // same, so a read of it reports the damage rather than + // answering that the key is absent + n := uint32(len(value)) + if !ok { + n = binary.LittleEndian.Uint32(buf[at:]) + } + if key != heapDeltaKey && !skip[heapRange{file: r.file, from: uint32(off)}] { + names = append(names, naming{key: key, s: slot{file: r.file, off: uint32(off), n: n}}) + } + at += size + } + } + for _, n := range names { + h.index[n.key] = n.s + } + h.nameRanges(ranges) + if height > h.height { + h.height = height + } + return true +} + +func (h *HeapStore) encodeIndexOf(magic uint32, each func(emit func(key [32]byte)), count int) []byte { + buf := make([]byte, heapIndexHdr+count*heapIndexRec+4) + binary.LittleEndian.PutUint32(buf, magic) + binary.LittleEndian.PutUint64(buf[4:], h.height) + binary.LittleEndian.PutUint32(buf[12:], uint32(count)) + binary.LittleEndian.PutUint32(buf[16:], h.deltaAt.file) + binary.LittleEndian.PutUint32(buf[20:], uint32(h.deltaAt.off)) + at := heapIndexHdr + each(func(key [32]byte) { + s := h.index[key] + copy(buf[at:], key[:]) + binary.LittleEndian.PutUint32(buf[at+32:], s.file) + binary.LittleEndian.PutUint32(buf[at+36:], s.off) + binary.LittleEndian.PutUint32(buf[at+40:], s.n) + at += heapIndexRec + }) + binary.LittleEndian.PutUint32(buf[at:], crc32.ChecksumIEEE(buf[:at])) + return buf +} + +var errHeapTorn = errors.New("heap: torn index record") + +// applyRecord applies one index record and returns its length. +func (h *HeapStore) applyRecord(buf []byte) (int, error) { + if len(buf) < heapIndexHdr { + return 0, errHeapTorn + } + magic := binary.LittleEndian.Uint32(buf) + if magic != heapSnapshot { + return 0, errHeapTorn + } + count := int(binary.LittleEndian.Uint32(buf[12:])) + end := heapIndexHdr + count*heapIndexRec + if len(buf) < end+4 || crc32.ChecksumIEEE(buf[:end]) != binary.LittleEndian.Uint32(buf[end:]) { + return 0, errHeapTorn + } + if height := binary.LittleEndian.Uint64(buf[4:]); height > h.height { + h.height = height + } + if magic == heapSnapshot { + h.deltaAt.file = binary.LittleEndian.Uint32(buf[16:]) + h.deltaAt.off = int64(binary.LittleEndian.Uint32(buf[20:])) + } + for at := heapIndexHdr; at < end; at += heapIndexRec { + var key [32]byte + copy(key[:], buf[at:]) + h.index[key] = slot{file: binary.LittleEndian.Uint32(buf[at+32:]), off: binary.LittleEndian.Uint32(buf[at+36:]), n: binary.LittleEndian.Uint32(buf[at+40:])} + } + return end + 4, nil +} + +// deriveFiles derives every open file's live and dead bytes, cuts +// the newest back to its last named slot or its last delta, and +// deletes any file the map does not name and no delta needed: +// nothing durable names it, and its bytes are a crash's or the +// mover's leftovers. The caller holds the lock. +func (h *HeapStore) deriveFiles(dataIDs []uint32) error { + named := map[uint32]int64{} // file -> end of its last named slot + for _, s := range h.index { + if e := int64(s.off) + entrySize(int(s.n)); e > named[s.file] { + named[s.file] = e + } + h.liveBytes += entrySize(int(s.n)) + } + for id := range named { + if h.files[id] == nil { + return fmt.Errorf("heap: the index names %s: missing", dataName(id)) + } + } + for _, id := range dataIDs { + hf := h.files[id] + end, live := named[id] + if id == h.deltaAt.file && h.deltaAt.off > end { + end, live = h.deltaAt.off, true + } + if hf.namedEnd > end { + end, live = hf.namedEnd, true // A delta of this generation names it + } + if !live && hf.deltas == 0 { + hf.f.Close() + delete(h.files, id) + os.Remove(filepath.Join(h.Directory, dataName(id))) + continue + } + if err := hf.f.Truncate(end); err != nil { + return err + } + hf.size = end + } + for _, s := range h.index { + h.files[s.file].live += entrySize(int(s.n)) + } + for _, hf := range h.files { + hf.live += hf.deltas + h.liveBytes += hf.deltas + hf.dead = hf.size - hf.live + h.deadBytes += hf.dead + } + return nil +} + +// Close syncs what is pending and closes the files. Reopen with Open. +func (h *HeapStore) Close() error { + p, err := h.beginBlockSync() + if err != nil { + if errors.Is(err, errStoreClosed) { + return nil + } + return err + } + if err := p.finish(); err != nil { + return err + } + if err := h.unlinkReleased(); err != nil { + return err + } + h.mu.Lock() + defer h.mu.Unlock() + h.closed = true + for _, hf := range h.files { + if cerr := hf.f.Close(); err == nil { + err = cerr + } + } + if cerr := h.log.Close(); err == nil { + err = cerr + } + h.files, h.cur, h.mov, h.log = nil, nil, nil, nil + return err +} + +// encodeEntry lays out one entry: the block that wrote it is in the +// header so that, with the index gone, a scan can tell the current +// copy of a key (the highest height) from stale ones and a committed +// entry from one whose block never synced. +func encodeEntry(height uint64, key [32]byte, value []byte) []byte { + buf := make([]byte, entrySize(len(value))) + binary.LittleEndian.PutUint32(buf, uint32(len(value))) + binary.LittleEndian.PutUint64(buf[4:], height) + copy(buf[12:], key[:]) + copy(buf[heapHeader:], value) + binary.LittleEndian.PutUint32(buf[heapHeader+len(value):], crc32.ChecksumIEEE(buf[4:heapHeader+len(value)])) + return buf +} + +// decodeEntry checks the entry at the start of buf and returns its +// fields and its aligned size; ok is false for an unwritten, torn or +// damaged slot. +func decodeEntry(buf []byte) (size int64, height uint64, key [32]byte, value []byte, ok bool) { + if len(buf) < heapHeader { + return + } + n := int(binary.LittleEndian.Uint32(buf)) + size = entrySize(n) + if n == 0 && binary.LittleEndian.Uint64(buf[4:]) == 0 || size > int64(len(buf)) { + return 0, 0, key, nil, false + } + copy(key[:], buf[12:]) + if crc32.ChecksumIEEE(buf[4:heapHeader+n]) != binary.LittleEndian.Uint32(buf[heapHeader+n:]) { + return size, 0, key, nil, false // Damaged: its size lets a scan step over it, its key lets replay name it + } + height = binary.LittleEndian.Uint64(buf[4:]) + copy(key[:], buf[12:]) + return size, height, key, buf[heapHeader : heapHeader+n], true +} + +// newFile opens the next data file. The caller holds the lock. +func (h *HeapStore) newFile() (*heapFile, error) { + id := h.nextID + h.nextID++ + f, err := os.OpenFile(filepath.Join(h.Directory, dataName(id)), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return nil, err + } + hf := &heapFile{id: id, f: f} + h.files[id] = hf + return hf, nil +} + +// reserve takes size bytes at the end of the block's or the mover's +// file, rolling to a new file at HeapFileBytes. The caller holds the +// lock. +func (h *HeapStore) reserve(mover bool, size int64) (hf *heapFile, off int64, err error) { + at := &h.cur + if mover { + at = &h.mov + } + if *at == nil || (*at).size+size > HeapFileBytes || (!mover && h.height >= (*at).firstBlock+HeapFileBlocks) { + if *at, err = h.newFile(); err != nil { + return nil, 0, err + } + (*at).firstBlock = h.height + } + hf = *at + off = hf.size + hf.size += size + hf.live += size + h.liveBytes += size + if mover { + // The pass syncs its own file; the block's sync must never + // capture it as dirty, or the block waits for the copies + hf.inflight++ + } else { + h.dirty[hf.id] = hf + h.extendRange(hf.id, off, off+size) + } + return hf, off, nil +} + +// extendRange adds [from, to) of a file to the block's delta, +// extending the last range when it abuts. The caller holds the lock. +func (h *HeapStore) extendRange(file uint32, from, to int64) { + if n := len(h.ranges); n > 0 && h.ranges[n-1].file == file && int64(h.ranges[n-1].to) == from { + h.ranges[n-1].to = uint32(to) + return + } + h.ranges = append(h.ranges, heapRange{file: file, from: uint32(from), to: uint32(to)}) +} + +// kill accounts a slot its key stopped naming. The caller holds the +// lock. +func (h *HeapStore) kill(s slot) { + size := entrySize(int(s.n)) + h.liveBytes -= size + h.deadBytes += size + hf := h.files[s.file] + hf.live -= size + hf.dead += size +} + +// Put writes value under key: in place if the key took its slot this +// block and the entry keeps its size, else appended to the block's +// file. The slot a durable index names is never rewritten. +func (h *HeapStore) Put(key [32]byte, value []byte) error { + h.mu.Lock() + defer h.mu.Unlock() + if h.closed { + return errStoreClosed + } + h.putTotal.Add(1) + old, had := h.index[key] + var s slot + var hf *heapFile + if had && old.block == h.height && entrySize(len(value)) == entrySize(int(old.n)) { + // Taken this block, nothing durable names it, and the same + // aligned size, so the file stays a contiguous sequence of + // entries a scan can walk: rewrite in place + s = old + s.n = uint32(len(value)) + hf = h.files[s.file] + h.dirty[hf.id] = hf + h.putInPlace.Add(1) + } else { + var off int64 + var err error + if hf, off, err = h.reserve(false, entrySize(len(value))); err != nil { + return err + } + s = slot{file: hf.id, off: uint32(off), n: uint32(len(value)), block: h.height} + if had { + h.kill(old) // Dead where it lies + } + h.putAppend.Add(1) + } + if _, err := hf.f.WriteAt(encodeEntry(h.height, key, value), int64(s.off)); err != nil { + return err + } + h.index[key] = s + return nil +} + +// Get answers from the key map and one read of the slot, taken +// outside the lock. A slot whose checksum fails is reported as +// corrupt, never as a value. A file deleted between the lookup and +// the read -- the mover moved the entry and a sync released the file +// meanwhile -- is looked up again. +func (h *HeapStore) Get(key [32]byte) ([]byte, error) { + for attempt := 0; ; attempt++ { + h.mu.RLock() + if h.closed { + h.mu.RUnlock() + return nil, errStoreClosed + } + h.lookups.Add(1) + s, ok := h.index[key] + var f *os.File + if ok { + f = h.files[s.file].f + } + h.mu.RUnlock() + if !ok { + return nil, errNotFound + } + h.hits.Add(1) + buf := make([]byte, heapHeader+int(s.n)+heapTrailer) + if _, err := f.ReadAt(buf, int64(s.off)); err != nil { + if attempt == 0 && errors.Is(err, os.ErrClosed) { + continue + } + return nil, err + } + return heapEntryValue(buf, key) + } +} + +// heapEntryValue checks an entry read from a slot and returns its +// value. +func heapEntryValue(buf []byte, key [32]byte) ([]byte, error) { + n := int(binary.LittleEndian.Uint32(buf)) + if len(buf) != heapHeader+n+heapTrailer || [32]byte(buf[12:heapHeader]) != key { + return nil, fmt.Errorf("heap: slot does not hold the key it is named for") + } + if crc32.ChecksumIEEE(buf[4:heapHeader+n]) != binary.LittleEndian.Uint32(buf[heapHeader+n:]) { + return nil, fmt.Errorf("heap: entry checksum failed") + } + return append([]byte(nil), buf[heapHeader:heapHeader+n]...), nil +} + +// GetDeep is Get: a heap has no history to reach into. +func (h *HeapStore) GetDeep(key [32]byte) ([]byte, error) { return h.Get(key) } + +// AdvanceBlock sets the block new writes belong to. +func (h *HeapStore) AdvanceBlock(height uint64) { + h.mu.Lock() + h.height = height + h.mu.Unlock() +} + +// heapSync is a block sync in flight: the files to make durable, the +// delta, and the files to delete once it is. It holds syncMu until +// finished. +type heapSync struct { + h *HeapStore + dirty []*heapFile + delta []byte + release []uint32 + bytes int64 +} + +// beginBlockSync takes the block's delta under the lock; finish makes +// it durable outside it. +func (h *HeapStore) beginBlockSync() (blockSync, error) { + h.syncMu.Lock() + h.mu.Lock() + defer h.mu.Unlock() + if h.closed || h.files == nil { + h.syncMu.Unlock() + return nil, errStoreClosed + } + p := &heapSync{h: h, release: h.release} + h.release = nil + if len(h.ranges) > 0 || len(p.release) > 0 { + // The delta goes into the block's data file behind its entries, + // under the reserved key, so the block's one fsync covers both + p.delta = h.encodeDelta() + h.nameRanges(h.ranges) + h.ranges, h.excluded = nil, nil + hf, off, err := h.reserve(false, entrySize(len(p.delta))) + h.ranges = nil // The delta entry itself is not part of the block's range + if err != nil { + h.syncMu.Unlock() + return nil, err + } + if _, err := hf.f.WriteAt(encodeEntry(h.height, heapDeltaKey, p.delta), off); err != nil { + h.syncMu.Unlock() + return nil, err + } + hf.deltas += entrySize(len(p.delta)) + h.deltaAt.file, h.deltaAt.off = hf.id, off+entrySize(len(p.delta)) + } + for _, hf := range h.dirty { + p.dirty = append(p.dirty, hf) + p.bytes += hf.size + } + h.dirty = map[uint32]*heapFile{} + return p, nil +} + +// finish: entries durable, then the delta durable, then the files the +// delta's copies emptied are deleted. Releases syncMu. +func (p *heapSync) finish() (err error) { + h := p.h + defer h.syncMu.Unlock() + if p.delta == nil { + return nil + } + t := time.Now() + for _, hf := range p.dirty { + if err = fsync(hf.f); err != nil { + return err + } + } + h.syncHeapNs.Add(uint64(time.Since(t))) + h.syncs.Add(1) + h.syncBytes.Add(uint64(p.bytes)) + h.mu.Lock() + defer h.mu.Unlock() + // The files the delta no longer names leave the map now; their + // unlinks are the mover's, off the block's path. Until then they + // are unnamed files on disk, which an open deletes as such. + for _, id := range p.release { + hf := h.files[id] + hf.f.Close() + delete(h.files, id) + h.deadBytes -= hf.dead + h.unlink = append(h.unlink, id) + } + return nil +} + +// SetSnapshotPhase offsets this shard's snapshot cadence by blocks, so +// that a store's shards do not all snapshot on the same block. +func (h *HeapStore) SetSnapshotPhase(blocks uint64) { + h.mu.Lock() + h.snapPhase = blocks + h.mu.Unlock() +} + +// unlinkReleased deletes the files the block syncs have released. +// Called without the lock. +func (h *HeapStore) unlinkReleased() error { + h.mu.Lock() + ids := h.unlink + h.unlink = nil + h.mu.Unlock() + if len(ids) == 0 { + return nil + } + t := time.Now() + for _, id := range ids { + if err := os.Remove(filepath.Join(h.Directory, dataName(id))); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + } + h.releases.Add(uint64(len(ids))) + h.releaseNs.Add(uint64(time.Since(t))) + return nil +} + +// compact is one slice of the heap's maintenance: the released +// files unlinked, a mover pass bounded by budget bytes copied, and +// every HeapSnapshotBlocks a new index generation, which bounds the +// replay on open. +func (h *HeapStore) compact(budget int64) (bool, error) { + if err := h.unlinkReleased(); err != nil { + return false, err + } + moved, err := h.clean(budget) + if err != nil { + return moved, err + } + // A snapshot is due by age, or when the files it would free + // outweigh it: a file whose only live bytes are deltas cannot go + // until a snapshot supersedes them, so the space those files hold + // is what a snapshot buys, and it is taken when that exceeds + // HeapSnapshotPinned + h.mu.Lock() + var pinned int64 + for _, hf := range h.files { + if hf != h.cur && hf != h.mov && hf.live == hf.deltas && (hf.deltas > 0 || hf.named > 0) { + pinned += hf.size // Nothing live in it but what the generation's deltas need + } + } + due := h.height+h.snapPhase-h.snapAt >= HeapSnapshotBlocks || pinned >= HeapSnapshotPinnedFiles*HeapFileBytes + if due { + h.snapAt = h.height + } + h.mu.Unlock() + if due { + t := time.Now() + err = h.Snapshot() + h.snapshotsN.Add(1) + h.snapshotNs.Add(uint64(time.Since(t))) + } + return moved, err +} + +// Snapshot starts a new index generation: the whole map, then the +// deltas to come. Serialized with block syncs, so the map is exactly +// the state of the last finished delta and no delta lands in the +// generation being retired. Off the protocol path. +func (h *HeapStore) Snapshot() error { + h.mu.RLock() + closed := h.closed + h.mu.RUnlock() + if closed { + return errStoreClosed + } + return h.startGeneration() +} + +// moverHook, when set, runs between the mover's copy and its naming: +// the window in which a put can make a copy dead on arrival. +var moverHook func() + +// heapMove is one live entry the mover copies: where it was, where +// it goes, and the bytes to write there. +type heapMove struct { + key [32]byte + from, to slot + hf *heapFile + height uint64 // The block the copy is written as + value []byte + entry []byte +} + +// heapPlanChunk is how many entries a pass plans per lock hold. +const heapPlanChunk = 4096 + +// clean is the mover: it takes the files with the most dead bytes -- +// once HeapCleanRatio of each is dead, or the deadest whatever its +// ratio while dead bytes exceed live -- copies their live entries to +// the mover's file, at most budget bytes per pass, and marks a file +// left with nothing live for deletion once the delta naming the +// copies is durable. The pass syncs its own copies, so they never +// land in a block's barrier, and it holds the shard's lock only to +// pick, to plan, and to name: never across a read, a write or an +// fsync (spec 1.6). A byte released never costs more than a byte +// copied unless the size bound forces it (spec 1.2). +func (h *HeapStore) clean(budget int64) (bool, error) { + // 1. Under the lock: pick the files + h.mu.Lock() + if h.closed { + h.mu.Unlock() + return false, errStoreClosed + } + if len(h.release) > 0 { + h.mu.Unlock() + return false, nil // The last pass's deletions are still waiting on a sync + } + // A file with nothing live in it needs no scan and no copy: it is + // released outright, any number of them a pass, one sync late as + // every release is. Without this the pass's file count paced the + // mover at about the rate dead bytes appeared, and the store + // floated with the size bound engaged. + emptied := 0 + for _, hf := range h.files { + if hf.live == 0 && hf.named == 0 && hf != h.cur && hf != h.mov && !hf.cleaning && hf.inflight == 0 && !hf.releasing && hf.size > 0 { + hf.releasing = true + h.release = append(h.release, hf.id) + emptied++ + } + } + var taken []*heapFile + var scan int64 + for len(taken) < HeapCleanFiles && scan < HeapScanBytes { + hf := h.pickFile() + if hf == nil { + break + } + hf.cleaning = true + taken = append(taken, hf) + scan += hf.size + } + h.mu.Unlock() + if len(taken) == 0 { + return emptied > 0, nil + } + // 2. Under the lock, briefly: the picked files' live slots, from + // the index -- one pass over the map, not a decode of the files. + // Scanning a file to find its live entries read and checksummed + // up to 128 MB a shard per pass, and nine stores' passes together + // starved the block loops for CPU (load 27 on 24 cores, seal p50 + // 200 ms for the minute). Then without the lock: read those + // entries, and only those + type liveSlot struct { + key [32]byte + s slot + } + h.mu.Lock() + picked := map[uint32]int{} + for i, hf := range taken { + picked[hf.id] = i + } + slots := make([][]liveSlot, len(taken)) + for key, s := range h.index { + if i, ok := picked[s.file]; ok { + slots[i] = append(slots[i], liveSlot{key: key, s: s}) + } + } + h.mu.Unlock() + entries := make([][]heapEntry, len(taken)) + for i, hf := range taken { + sort.Slice(slots[i], func(a, b int) bool { return slots[i][a].s.off < slots[i][b].s.off }) + for _, ls := range slots[i] { + buf := make([]byte, entrySize(int(ls.s.n))) + if _, err := hf.f.ReadAt(buf, int64(ls.s.off)); err != nil { + h.mu.Lock() + for _, hf := range taken { + hf.cleaning = false + } + h.mu.Unlock() + return false, err + } + size, _, key, value, ok := decodeEntry(buf) + if !ok || key != ls.key { + continue // Damaged or stale: left where it is, and a read of it reports the damage + } + entries[i] = append(entries[i], heapEntry{off: ls.s.off, size: size, key: key, value: value}) + } + h.cleanedBytes.Add(uint64(hf.size)) + } + // 3. Under the lock, a chunk of entries at a time: confirm each is + // still named and reserve its copy's slot in the mover's file + var moves []heapMove + var copied int64 + for i, hf := range taken { + for at := 0; at < len(entries[i]) && copied < budget; at += heapPlanChunk { + end := at + heapPlanChunk + if end > len(entries[i]) { + end = len(entries[i]) + } + h.mu.Lock() + m, n, err := h.planFile(hf, entries[i][at:end], budget-copied) + h.mu.Unlock() + if err != nil { + h.mu.Lock() + for _, hf := range taken { + hf.cleaning = false + } + h.mu.Unlock() + return false, err + } + moves = append(moves, m...) + copied += n + } + } + h.mu.Lock() + for _, hf := range taken { + hf.cleaning = false + } + h.mu.Unlock() + var movFiles []*heapFile + for _, m := range moves { + if len(movFiles) == 0 || movFiles[len(movFiles)-1] != m.hf { + movFiles = append(movFiles, m.hf) + } + } + // The copies' bytes are laid out outside the lock + for i := range moves { + m := &moves[i] + m.entry = encodeEntry(m.height, m.key, m.value) + } + // 4. Without the lock: write the copies and make them durable + fail := func(err error) (bool, error) { + h.mu.Lock() + for _, m := range moves { + m.hf.inflight-- // Reserved, never written: dead, and read no further + h.kill(m.to) + } + h.mu.Unlock() + return false, err + } + for _, m := range moves { + if _, err := m.hf.f.WriteAt(m.entry, int64(m.to.off)); err != nil { + return fail(err) + } + } + for _, hf := range movFiles { + if err := fsync(hf.f); err != nil { + return fail(err) + } + } + if moverHook != nil { + moverHook() // Tests: a put between the copy and its naming + } + // 5. Under the lock: name the copies, unless the key was rewritten + // meanwhile, in which case the copy is dead on arrival; and mark + // the files the pass emptied + h.mu.Lock() + defer h.mu.Unlock() + // The copies were reserved in order, so they form ranges of the + // mover's files; a copy that arrived dead is excluded from them. + // The mover's ranges go before the block's own in the delta, so + // that a put of the same key later in the block wins on replay + var mine []heapRange + for _, m := range moves { + m.hf.inflight-- + size := entrySize(int(m.to.n)) + if n := len(mine); n > 0 && mine[n-1].file == m.to.file && int64(mine[n-1].to) == int64(m.to.off) { + mine[n-1].to = uint32(int64(m.to.off) + size) + } else { + mine = append(mine, heapRange{file: m.to.file, from: m.to.off, to: uint32(int64(m.to.off) + size)}) + } + s, live := h.index[m.key] + if live && s == m.from { + h.index[m.key] = m.to + h.kill(m.from) + } else { + h.kill(m.to) // Reserved and written, but no longer wanted + h.excluded = append(h.excluded, heapRange{file: m.to.file, from: m.to.off}) + } + } + h.ranges = append(mine, h.ranges...) + for _, hf := range taken { + if hf.live == 0 && hf.named == 0 && hf != h.cur && hf != h.mov { + hf.releasing = true + h.release = append(h.release, hf.id) + } + } + h.movedBytes.Add(uint64(copied)) + return true, nil +} + +// pickFile is the file the mover takes next: the deadest by fraction +// among those not open for append, if it is dead enough or the heap +// is over its size bound. The caller holds the lock. +func (h *HeapStore) pickFile() *heapFile { + var pick *heapFile + best := 0.0 + for _, hf := range h.files { + if hf == h.cur || hf == h.mov || hf.cleaning || hf.inflight > 0 || hf.releasing || hf.dead == 0 { + continue + } + if f := float64(hf.dead) / float64(hf.size); f > best { + pick, best = hf, f + } + } + switch { + case !h.bound && float64(h.deadBytes) > HeapBoundOn*float64(h.liveBytes): + h.bound = true + case h.bound && h.deadBytes < h.liveBytes: + h.bound = false + } + if pick == nil || best < HeapCleanRatio && (!h.bound || best < HeapCleanFloor) { + return nil + } + return pick +} + +// heapEntry is one entry of a picked file as decoded outside the +// lock: where it is, and its bytes ready to be copied. +type heapEntry struct { + off uint32 + size int64 + key [32]byte + value []byte +} + +// planFile reserves, in the mover's file, a slot for each entry of a +// picked file the index still names, up to budget bytes; the rest +// wait for the next pass. The caller holds the lock; the entries +// were decoded outside it. +func (h *HeapStore) planFile(hf *heapFile, entries []heapEntry, budget int64) (moves []heapMove, copied int64, err error) { + for _, e := range entries { + s, live := h.index[e.key] + if !live || s.file != hf.id || s.off != e.off { + continue + } + if copied >= budget { + break // The rest next pass + } + to, off, err := h.reserve(true, e.size) + if err != nil { + return nil, 0, err + } + ns := slot{file: to.id, off: uint32(off), n: uint32(len(e.value)), block: h.height} + // The copy is this block's write of the key: it carries this + // height, so a repair scan prefers it to the original + moves = append(moves, heapMove{key: e.key, from: s, to: ns, hf: to, height: h.height, value: e.value}) + copied += e.size + } + return moves, copied, nil +} + +// RepairHeapStore rebuilds the key map by reading the keys from the +// data: every entry carries its key, its height and a checksum, so a +// scan of the data files in order recovers the map without any index. +// For each key the copy with the highest height wins, later in the +// scan on a tie; an entry above the committed height -- a block whose +// sync never finished -- is dropped, as is any torn or damaged slot. +// The rebuilt map starts a new generation, so the next open is an +// ordinary one. +func RepairHeapStore(directory string, committed uint64) (*HeapStore, error) { + dataIDs, gens, err := listHeap(directory) + if err != nil { + return nil, err + } + for _, g := range gens { + os.Remove(filepath.Join(directory, indexName(g))) + } + h := &HeapStore{Directory: directory, index: map[[32]byte]slot{}, dirty: map[uint32]*heapFile{}, files: map[uint32]*heapFile{}, height: committed} + heights := map[[32]byte]uint64{} + for _, id := range dataIDs { + data, err := os.ReadFile(filepath.Join(directory, dataName(id))) + if err != nil { + return nil, err + } + var off int64 + for off < int64(len(data)) { + size, height, key, value, ok := decodeEntry(data[off:]) + if size == 0 { + break + } + if !ok || key == heapDeltaKey { + off += size + continue + } + if height <= committed { + if prev, seen := heights[key]; !seen || height >= prev { + heights[key] = height + h.index[key] = slot{file: id, off: uint32(off), n: uint32(len(value))} + } + } + off += size + } + if id >= h.nextID { + h.nextID = id + 1 + } + } + // A repair trusts the data alone: the snapshot's replay point is + // the end of the newest file, so no old delta is replayed over it + if n := len(dataIDs); n > 0 { + if st, err := os.Stat(filepath.Join(directory, dataName(dataIDs[n-1]))); err == nil { + h.deltaAt.file, h.deltaAt.off = dataIDs[n-1], st.Size() + } + } + h.gen = 1 + if err = h.startGeneration(); err != nil { + return nil, err + } + if err = h.openFiles(dataIDs); err != nil { + return nil, err + } + if err = h.deriveFiles(dataIDs); err != nil { + return nil, err + } + return h, nil +} + +// Stats is the heap's report in the store's terms: puts and lookups +// as the segment layer counts them (every hit is answered from the +// key map, so they are all LiveHit), and the heap's own figures. +func (h *HeapStore) Stats() StoreStats { + h.mu.RLock() + defer h.mu.RUnlock() + return StoreStats{ + PutTotal: h.putTotal.Load(), + PutNew: h.putAppend.Load(), + LookupTotal: h.lookups.Load(), + LiveHit: h.hits.Load(), + ResidentIndexBytes: uint64(len(h.index)) * (32 + 24), + HeapFiles: len(h.files), + HeapLiveBytes: uint64(h.liveBytes), + HeapDeadBytes: uint64(h.deadBytes), + HeapScannedBytes: h.cleanedBytes.Load(), + HeapMovedBytes: h.movedBytes.Load(), + } +} + +// HoleRatio reports the dead bytes in the files against the live +// bytes: what the mover has yet to reclaim. Live includes the delta +// entries a snapshot has not yet superseded. +func (h *HeapStore) HoleRatio() (dead, live int64) { + h.mu.RLock() + defer h.mu.RUnlock() + return h.deadBytes, h.liveBytes +} + +// deltaBytes is the bytes of delta entries not yet superseded. +func (h *HeapStore) deltaBytes() (n int64) { + h.mu.RLock() + defer h.mu.RUnlock() + for _, hf := range h.files { + n += hf.deltas + } + return n +} + +// SyncCost reports the block syncs so far: how many, the bytes their +// data fsyncs covered, and the time spent in the data files' fsyncs +// and in the delta's write and fsync. +func (h *HeapStore) SyncCost() (syncs, bytes uint64, heapFsync, delta time.Duration) { + return h.syncs.Load(), h.syncBytes.Load(), time.Duration(h.syncHeapNs.Load()), time.Duration(h.syncLogNs.Load()) +} + +// Cleaned reports what the mover has scanned and what it had to +// copy: the ratio is the heap's write amplification. +func (h *HeapStore) Cleaned() (scanned, moved uint64) { + return h.cleanedBytes.Load(), h.movedBytes.Load() +} + +// MoverCost reports the files a block's finish has unlinked and the +// time in those unlinks, and the snapshots written and their time. +func (h *HeapStore) MoverCost() (releases uint64, release time.Duration, snapshots uint64, snapshot time.Duration) { + return h.releases.Load(), time.Duration(h.releaseNs.Load()), h.snapshotsN.Load(), time.Duration(h.snapshotNs.Load()) +} + +// SetFilterBlocks and SetSealLimit are the segment layer's knobs; a +// heap has neither a window nor a tail. +func (h *HeapStore) SetFilterBlocks(uint64) error { return nil } +func (h *HeapStore) SetSealLimit(uint64) error { return nil } + +// LiveRecords is the live key count; a heap is never sealed on it. +func (h *HeapStore) LiveRecords() uint64 { + h.mu.RLock() + defer h.mu.RUnlock() + return uint64(len(h.index)) +} diff --git a/database/heap_crash_test.go b/database/heap_crash_test.go new file mode 100644 index 0000000..1a469bb --- /dev/null +++ b/database/heap_crash_test.go @@ -0,0 +1,123 @@ +package blockchainDB + +import ( + "bufio" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +const heapCrashDirEnv = "BDB_HEAP_CRASH_DIR" + +// The child: blocks of the same keys, every block rewriting every key +// with a value that says which block, a sync per block and a mover +// pass every few, and a line on stdout at each durability point. +// Killed by the parent at a moment of its choosing. +func TestHeapCrashChild(t *testing.T) { + dir := os.Getenv(heapCrashDirEnv) + if dir == "" { + t.Skip("helper process for TestHeapCrashRecovery") + } + HeapFileBytes = 64 << 10 + h, err := NewHeapStore(dir) + require.NoError(t, err, "child: new") + for b := uint64(1); ; b++ { + h.AdvanceBlock(b) + for i := 0; i < 300; i++ { + require.NoError(t, h.Put(crashKey(i), crashValue(b, i)), "child: put") + } + p, err := h.beginBlockSync() + require.NoError(t, err, "child: sync") + require.NoError(t, p.finish(), "child: finish") + fmt.Printf("CHECKPOINT %d\n", b) + if b%4 == 0 { + _, err := h.compact(HeapCleanBytes) + require.NoError(t, err, "child: compact") + } + } +} + +func crashKey(i int) (k [32]byte) { + k[0], k[1] = byte(i>>8), byte(i) + return +} + +func crashValue(b uint64, i int) []byte { + v := make([]byte, 40+i%60) + v[0], v[1], v[2] = byte(b>>16), byte(b>>8), byte(b) + v[3] = byte(i) + return v +} + +// The parent: run the child, let it reach some durability points, +// kill it dead, and check that the heap opens to exactly the last +// durable block -- every key at that block's value, nothing torn, +// nothing from the block in flight -- and that a repair from the data +// alone agrees. +func TestHeapCrashRecovery(t *testing.T) { + if testing.Short() { + t.Skip("spawns processes") + } + for round := 0; round < 3; round++ { + dir := filepath.Join(t.TempDir(), fmt.Sprintf("crash-%d", round)) + cmd := exec.Command(os.Args[0], "-test.run", "TestHeapCrashChild$") + cmd.Env = append(os.Environ(), heapCrashDirEnv+"="+dir) + out, err := cmd.StdoutPipe() + require.NoError(t, err) + require.NoError(t, cmd.Start()) + var last uint64 + sc := bufio.NewScanner(out) + want := uint64(6 + round*5) + for sc.Scan() { + line := sc.Text() + if !strings.HasPrefix(line, "CHECKPOINT ") { + continue + } + n, err := strconv.ParseUint(strings.TrimPrefix(line, "CHECKPOINT "), 10, 64) + require.NoError(t, err) + last = n + if last >= want { + break + } + } + // Somewhere inside the next block: the child is mid-write or + // mid-sync when it dies + time.Sleep(time.Duration(round*7) * time.Millisecond) + require.NoError(t, cmd.Process.Kill()) + _ = cmd.Wait() + + check := func(h *HeapStore, what string) { + t.Helper() + for i := 0; i < 300; i++ { + v, err := h.Get(crashKey(i)) + require.NoError(t, err, "%s: key %d", what, i) + require.Equal(t, crashValue(h.height, i), v, "%s: key %d is at the durable block", what, i) + } + } + h, err := OpenHeapStore(dir) + require.NoError(t, err) + require.GreaterOrEqual(t, h.height, last, "the durable block is at least the last checkpoint the parent saw") + check(h, "open") + durable := h.height + require.NoError(t, h.Close()) + + // The index thrown away: the data alone must say the same + _, gens, err := listHeap(dir) + require.NoError(t, err) + for _, g := range gens { + require.NoError(t, os.Remove(filepath.Join(dir, indexName(g)))) + } + r, err := RepairHeapStore(dir, durable) + require.NoError(t, err) + require.Equal(t, durable, r.height) + check(r, "repair") + require.NoError(t, r.Close()) + } +} diff --git a/database/heap_test.go b/database/heap_test.go new file mode 100644 index 0000000..4577d88 --- /dev/null +++ b/database/heap_test.go @@ -0,0 +1,451 @@ +package blockchainDB + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func heapDir(t *testing.T) string { + t.Helper() + return filepath.Join(t.TempDir(), "heap") +} + +func key(b byte) (k [32]byte) { k[0] = b; return } + +func syncHeap(t *testing.T, h *HeapStore) { + t.Helper() + p, err := h.beginBlockSync() + require.NoError(t, err) + require.NoError(t, p.finish()) +} + +// Small files, so that rolling and deletion happen within a test. +func smallFiles(t *testing.T) { + t.Helper() + was := HeapFileBytes + HeapFileBytes = 1024 + t.Cleanup(func() { HeapFileBytes = was }) +} + +// A key rewritten within the block reuses its slot; rewritten in a +// later block it is appended, the old slot dead where it lies until +// the mover copies the file's live entries out and the next sync +// deletes the file. +func TestHeapRewriteReusesWithinTheBlockAndMovesOneSyncLate(t *testing.T) { + smallFiles(t) + h, err := NewHeapStore(heapDir(t)) + require.NoError(t, err) + defer h.Close() + h.AdvanceBlock(1) + require.NoError(t, h.Put(key(1), []byte("one"))) + first := h.index[key(1)] + require.NoError(t, h.Put(key(1), []byte("uno"))) + require.Equal(t, first, h.index[key(1)], "same block, fits: in place") + require.EqualValues(t, 1, h.putInPlace.Load()) + v, err := h.Get(key(1)) + require.NoError(t, err) + require.Equal(t, "uno", string(v)) + + syncHeap(t, h) + h.AdvanceBlock(2) + require.NoError(t, h.Put(key(1), []byte("two"))) + require.NotEqual(t, first, h.index[key(1)], "a durable slot is never rewritten") + dead, live := h.HoleRatio() + require.EqualValues(t, entrySize(3), dead, "the old slot is dead where it lies") + require.EqualValues(t, entrySize(3), live-h.deltaBytes(), "one live entry besides the deltas") + // Fill the block's file past its size so it rolls, then kill most + // of what the first file holds + for i := byte(10); i < 40; i++ { + require.NoError(t, h.Put(key(i), make([]byte, 64))) + } + syncHeap(t, h) + require.Greater(t, len(h.files), 1, "the block's file rolled") + h.AdvanceBlock(3) + for i := byte(10); i < 39; i++ { + require.NoError(t, h.Put(key(i), make([]byte, 64))) + } + syncHeap(t, h) + require.NoError(t, h.Snapshot()) // The deltas so far are superseded: dead where they lie + + h.AdvanceBlock(4) + moved, err := h.clean(1 << 20) + require.NoError(t, err) + require.True(t, moved, "a mostly dead file is taken") + require.NotEmpty(t, h.release, "a file emptied by the pass waits for the sync") + for _, id := range h.release { + _, err := os.Stat(filepath.Join(h.Directory, dataName(id))) + require.NoError(t, err, "not deleted yet: the copies are not durable") + } + released := append([]uint32(nil), h.release...) + syncHeap(t, h) + for _, id := range released { + require.Nil(t, h.files[id], "out of the map after the sync") + _, err := os.Stat(filepath.Join(h.Directory, dataName(id))) + require.NoError(t, err, "the unlink is the mover's, off the block's path") + } + require.NoError(t, h.unlinkReleased()) + for _, id := range released { + _, err := os.Stat(filepath.Join(h.Directory, dataName(id))) + require.ErrorIs(t, err, os.ErrNotExist, "deleted by the mover") + } + _, copied := h.Cleaned() + require.Greater(t, copied, uint64(0), "the live entries left in it were copied out") + v, err = h.Get(key(1)) + require.NoError(t, err) + require.Equal(t, "two", string(v)) + for i := byte(10); i < 40; i++ { + _, err := h.Get(key(i)) + require.NoError(t, err) + } +} + +// Reopening replays the generation: every synced value is back, the +// files' accounting is derived, and a block that was never synced is +// gone -- its slots unnamed and its bytes cut from the file. +func TestHeapReopenKeepsTheDurableAndDropsTheRest(t *testing.T) { + dir := heapDir(t) + h, err := NewHeapStore(dir) + require.NoError(t, err) + h.AdvanceBlock(1) + for i := byte(1); i <= 50; i++ { + require.NoError(t, h.Put(key(i), []byte{i})) + } + syncHeap(t, h) + h.AdvanceBlock(2) + require.NoError(t, h.Put(key(1), []byte("rewritten in block 2"))) + syncHeap(t, h) + size := h.deltaAt.off // Where block 2's delta ended: the durable append point + // Block 3: written, never synced -- the crash + h.AdvanceBlock(3) + require.NoError(t, h.Put(key(2), []byte("lost"))) + require.NoError(t, h.Put(key(99), []byte("lost too"))) + for _, hf := range h.files { + hf.f.Close() + } + h.log.Close() + + r, err := OpenHeapStore(dir) + require.NoError(t, err) + defer r.Close() + v, err := r.Get(key(1)) + require.NoError(t, err) + require.Equal(t, "rewritten in block 2", string(v)) + v, err = r.Get(key(2)) + require.NoError(t, err) + require.Equal(t, []byte{2}, v, "block 3's rewrite was never durable") + _, err = r.Get(key(99)) + require.ErrorIs(t, err, errNotFound) + require.Len(t, r.files, 1) + for _, hf := range r.files { + require.Equal(t, size, hf.size, "the file is cut back to the durable append point") + } + dead, _ := r.HoleRatio() + require.EqualValues(t, entrySize(1), dead, "key 1's block-1 slot is dead where it lies") + require.EqualValues(t, 2, r.height, "the durable height: block 3 never synced") +} + +// A torn entry at the end of the data file -- a crash mid-write -- +// is cut, and the deltas before it stand. +func TestHeapTornTailIsCut(t *testing.T) { + dir := heapDir(t) + h, err := NewHeapStore(dir) + require.NoError(t, err) + h.AdvanceBlock(1) + require.NoError(t, h.Put(key(1), []byte("one"))) + syncHeap(t, h) + end := h.deltaAt.off + require.NoError(t, h.Close()) + f, err := os.OpenFile(filepath.Join(dir, dataName(0)), os.O_WRONLY|os.O_APPEND, 0o644) + require.NoError(t, err) + _, err = f.Write([]byte{9, 0, 0, 0, 1, 2, 3, 4, 5, 6}) // A length and a few bytes of nothing + require.NoError(t, err) + require.NoError(t, f.Close()) + + r, err := OpenHeapStore(dir) + require.NoError(t, err) + defer r.Close() + v, err := r.Get(key(1)) + require.NoError(t, err) + require.Equal(t, "one", string(v)) + st, err := os.Stat(filepath.Join(dir, dataName(0))) + require.NoError(t, err) + require.Equal(t, end, st.Size(), "cut back to the last delta") +} + +// A snapshot starts a new generation in its own file and retires the +// old one; the replay lands on the new one; and a generation whose +// write was interrupted is ignored. +func TestHeapSnapshotStartsAGenerationSafely(t *testing.T) { + dir := heapDir(t) + h, err := NewHeapStore(dir) + require.NoError(t, err) + every := HeapSnapshotBlocks + HeapSnapshotBlocks = 1 + defer func() { HeapSnapshotBlocks = every }() + for b := uint64(1); b <= 30; b++ { + h.AdvanceBlock(b) + for i := byte(1); i <= 20; i++ { + require.NoError(t, h.Put(key(i), []byte{byte(b), i})) + } + syncHeap(t, h) + if b == 20 { + _, err := h.compact(HeapCleanBytes) + require.NoError(t, err) + _, err = os.Stat(filepath.Join(dir, indexName(1))) + require.ErrorIs(t, err, os.ErrNotExist, "generation 1 retired") + _, err = os.Stat(filepath.Join(dir, indexName(2))) + require.NoError(t, err) + } + } + require.NoError(t, h.Close()) + // A crash mid-snapshot leaves a .tmp; it is not a generation + require.NoError(t, os.WriteFile(filepath.Join(dir, indexName(3)+".tmp"), []byte("half"), 0o644)) + r, err := OpenHeapStore(dir) + require.NoError(t, err) + defer r.Close() + require.EqualValues(t, 2, r.gen) + _, err = os.Stat(filepath.Join(dir, indexName(3)+".tmp")) + require.ErrorIs(t, err, os.ErrNotExist) + for i := byte(1); i <= 20; i++ { + v, err := r.Get(key(i)) + require.NoError(t, err) + require.Equal(t, []byte{30, i}, v) + } +} + +// A slot whose bytes were damaged is an error, never a value. +func TestHeapChecksumCatchesADamagedSlot(t *testing.T) { + dir := heapDir(t) + h, err := NewHeapStore(dir) + require.NoError(t, err) + h.AdvanceBlock(1) + require.NoError(t, h.Put(key(1), []byte("intact"))) + s := h.index[key(1)] + syncHeap(t, h) + h.AdvanceBlock(2) + require.NoError(t, h.Put(key(2), []byte("later"))) // A later block's sync covers block 1 + require.NoError(t, h.Close()) + f, err := os.OpenFile(filepath.Join(dir, dataName(s.file)), os.O_WRONLY, 0o644) + require.NoError(t, err) + _, err = f.WriteAt([]byte("damaged"), int64(s.off)+heapHeader) + require.NoError(t, err) + require.NoError(t, f.Close()) + r, err := OpenHeapStore(dir) + require.NoError(t, err) + defer r.Close() + _, err = r.Get(key(1)) + require.ErrorContains(t, err, "checksum") +} + +// With the index gone, the map is rebuilt from the data: the highest +// committed copy of each key wins, an entry from the block that never +// synced is dropped. +func TestHeapRepairReadsTheKeysFromTheData(t *testing.T) { + smallFiles(t) + dir := heapDir(t) + h, err := NewHeapStore(dir) + require.NoError(t, err) + h.AdvanceBlock(1) + for i := byte(1); i <= 10; i++ { + require.NoError(t, h.Put(key(i), make([]byte, 100))) + } + syncHeap(t, h) + h.AdvanceBlock(2) + require.NoError(t, h.Put(key(1), []byte{2, 1})) // Newer copy, in a later file + syncHeap(t, h) + h.AdvanceBlock(3) + require.NoError(t, h.Put(key(2), []byte{3, 2})) // Never synced: not committed + for _, hf := range h.files { + hf.f.Close() + } + h.log.Close() + require.NoError(t, os.Remove(filepath.Join(dir, indexName(1)))) + + _, err = OpenHeapStore(dir) + require.ErrorIs(t, err, ErrHeapNeedsRepair) + r, err := RepairHeapStore(dir, 2) + require.NoError(t, err) + defer r.Close() + v, err := r.Get(key(1)) + require.NoError(t, err) + require.Equal(t, []byte{2, 1}, v, "the block-2 copy wins") + v, err = r.Get(key(2)) + require.NoError(t, err) + require.Len(t, v, 100, "block 3 never committed: its copy is dropped") + require.EqualValues(t, 10, r.LiveRecords()) + require.NoError(t, r.Close()) + re, err := OpenHeapStore(dir) + require.NoError(t, err, "the repair left a generation: an ordinary open") + defer re.Close() + v, err = re.Get(key(1)) + require.NoError(t, err) + require.Equal(t, []byte{2, 1}, v) +} + +// A shard built with the heap seals, compacts, closes and reopens as +// a heap, through the same KV2 and KVShard surface, with files rolled +// and deleted along the way. +func TestHeapShardRoundTrip(t *testing.T) { + was := HeapFileBytes + HeapFileBytes = 256 << 10 + defer func() { HeapFileBytes = was }() + dir := filepath.Join(t.TempDir(), "shards") + kvs, err := NewKVShardHeapN(dir, 2, 1000) + require.NoError(t, err) + require.NoError(t, kvs.SetFilterBlocks(MinFilterBlocks)) + fr := NewFastRandom([]byte{7}) + hot := make([][32]byte, 200) + for i := range hot { + hot[i] = fr.NextHash() + } + for b := uint64(1); b <= 60; b++ { + for _, k := range hot { + require.NoError(t, kvs.PutDyna(k, append([]byte{byte(b)}, fr.RandBuff(100, 300)...))) + } + require.NoError(t, kvs.PutPerm(fr.NextHash(), []byte("perm"))) + require.NoError(t, kvs.SealBlock(b)) + if b%10 == 0 { + require.NoError(t, kvs.Compress()) + if b > MinFilterBlocks { + _, err := kvs.MergeFinalized(b - MinFilterBlocks) + require.NoError(t, err) + } + } + } + for _, k := range hot { + v, err := kvs.GetDyna(k) + require.NoError(t, err) + require.Equal(t, byte(60), v[0]) + } + _, dyna := kvs.Stats() + require.EqualValues(t, 60*200, dyna.PutTotal) + // Maintenance is a slice per call, sized by the blocks since the + // last: a call a block over a few blocks visits every shard with + // budget to spare for what the last ten blocks left dead, and each + // seal releases what the pass before it emptied + for b := uint64(61); b <= 64; b++ { + require.NoError(t, kvs.SealBlock(b)) + require.NoError(t, kvs.Compress()) + } + require.NoError(t, kvs.SealBlock(65)) + dead, live := kvs.Shards[0].Heap.HoleRatio() + require.Less(t, dead, 2*live+HeapFileBytes, "dead bytes are bounded: at most the current file, which the mover never takes, beyond the live set") + require.NoError(t, kvs.Close()) + + re, err := OpenKVShard(dir) + require.NoError(t, err) + defer re.Close() + require.NotNil(t, re.Shards[0].Heap, "reopened as a heap") + require.Nil(t, re.Shards[0].DynaKV) + for _, k := range hot { + v, err := re.GetDyna(k) + require.NoError(t, err) + require.Equal(t, byte(60), v[0]) + } +} + +// A key rewritten between the mover's copy and its naming leaves the +// copy dead on arrival: the put's value stands, and the copy's bytes +// are accounted dead in the mover's file. +func TestHeapMoveIsDeadOnArrivalIfTheKeyWasRewritten(t *testing.T) { + smallFiles(t) // Nine 112-byte entries to a file + h, err := NewHeapStore(heapDir(t)) + require.NoError(t, err) + defer h.Close() + h.AdvanceBlock(1) + for i := byte(1); i <= 20; i++ { + require.NoError(t, h.Put(key(i), make([]byte, 64))) + } + syncHeap(t, h) + // Two more blocks rewriting every key but 20 leave key 20 the one + // live entry in a file otherwise dead: the mover's next pick + for b := uint64(2); b <= 3; b++ { + h.AdvanceBlock(b) + for i := byte(1); i <= 19; i++ { + require.NoError(t, h.Put(key(i), make([]byte, 64))) + } + syncHeap(t, h) + } + require.NoError(t, h.Snapshot()) // The deltas so far are superseded: the file is mostly dead + files := HeapCleanFiles + HeapCleanFiles = 64 // One pass takes every eligible file, key 20's half-dead one included + defer func() { HeapCleanFiles = files }() + h.AdvanceBlock(4) + moverHook = func() { + require.NoError(t, h.Put(key(20), []byte("rewritten while moving"))) + } + defer func() { moverHook = nil }() + deadBefore, _ := h.HoleRatio() + moved, err := h.clean(1 << 20) + require.NoError(t, err) + require.True(t, moved) + _, copied := h.Cleaned() + require.GreaterOrEqual(t, copied, uint64(entrySize(64)), "key 20 was among the entries copied") + v, err := h.Get(key(20)) + require.NoError(t, err) + require.Equal(t, "rewritten while moving", string(v), "the put wins") + // Every copied entry left its old slot dead; key 20's copy is dead + // as well, since the put took the key elsewhere before it was named + dead, _ := h.HoleRatio() + require.EqualValues(t, deadBefore+int64(copied)+entrySize(64), dead, "the old slots and the unwanted copy are dead") + syncHeap(t, h) + v, err = h.Get(key(20)) + require.NoError(t, err) + require.Equal(t, "rewritten while moving", string(v)) +} + +// A file the mover copied into is named by the deltas of the current +// generation, as the destination of their ranges. When every copy in +// it dies it must not be unlinked until a snapshot supersedes those +// deltas: never unlink what a durable index names (spec 1.7). Found +// by the platform's reopen check: a store closed cleanly would not +// open, "the index names heap-000006.dat: missing". +func TestHeapMoverFileNamedByDeltasOutlivesItsEntries(t *testing.T) { + was, blocks, pinned := HeapFileBytes, HeapSnapshotBlocks, HeapSnapshotPinnedFiles + HeapFileBytes, HeapSnapshotBlocks, HeapSnapshotPinnedFiles = 64<<10, 1<<20, 1<<30 // No snapshot at all + defer func() { HeapFileBytes, HeapSnapshotBlocks, HeapSnapshotPinnedFiles = was, blocks, pinned }() + dir := heapDir(t) + h, err := NewHeapStore(dir) + require.NoError(t, err) + value := make([]byte, 400) + last := map[byte]byte{} + // Half the keys are rewritten every block, half every seventh: a + // file is soon half dead with live entries the mover copies out, + // and the copies die within seven blocks, emptying the mover's file + put := func(b uint64) { + h.AdvanceBlock(b) + for i := byte(1); i <= 100; i++ { + if i > 50 && b%7 != 0 { + continue + } + value[0] = byte(b) + require.NoError(t, h.Put(key(i), value)) + last[i] = byte(b) + } + syncHeap(t, h) + } + for b := uint64(1); b <= 70; b++ { + put(b) + if b%3 == 0 { + _, err := h.compact(HeapCleanBytes) // Copies live entries into mover files, releases what emptied + require.NoError(t, err) + } + } + _, moved := h.Cleaned() + require.Greater(t, moved, uint64(0), "the mover must have copied") + releases, _, snapshots, _ := h.MoverCost() + t.Logf("moved %d bytes, %d files unlinked, %d snapshots, %d files open, gen %d", moved, releases, snapshots, len(h.files), h.gen) + require.NoError(t, h.Close()) + h, err = OpenHeapStore(dir) + require.NoError(t, err, "a store closed cleanly reopens") + t.Logf("after open: %d files, gen %d, replay point %v", len(h.files), h.gen, h.deltaAt) + for i := byte(1); i <= 100; i++ { + v, err := h.Get(key(i)) + require.NoError(t, err, "key %d", i) + require.Equal(t, last[i], v[0], "key %d", i) + } + require.NoError(t, h.Close()) +} diff --git a/database/kv_2.go b/database/kv_2.go index f5008cd..37c0d7e 100644 --- a/database/kv_2.go +++ b/database/kv_2.go @@ -13,6 +13,8 @@ import ( const PermDirName = "perm" const DynaDirName = "dyna" +const HeapDirName = "dyna-heap" // The dynamic layer as a heap with holes (heap.go) +const PermFilesDirName = "perm-files" // The permanent layer as files of entries and buckets of keys (perm.go) // KV2 // Maintains 2 layers of key value pairs with different immutability characteristics: @@ -58,8 +60,10 @@ type KV2 struct { // every commit and read of the shard for the whole copy (issue #57). Mutex sync.RWMutex Directory string // Directory where the PermKV and DynaKV directories are - PermKV *SegmentStore // The Perm layer: sealed, immutable segments - DynaKV *SegmentStore // The Dyna layer: sealed, mutable segments + PermKV *SegmentStore // The Perm layer: sealed, immutable segments; nil when Perm is set + Perm *PermStore // The Perm layer as files of entries and buckets of keys (perm.go); nil when PermKV is set + DynaKV *SegmentStore // The Dyna layer: sealed, mutable segments; nil when Heap is set + Heap *HeapStore // The Dyna layer as a heap with holes (heap.go); nil when DynaKV is set // dWrites and pWrites count writes to each layer since the last // Compress. Atomic, because the writes that bump them run // concurrently: KV2.Put takes the lock SHARED (see Put). @@ -150,20 +154,123 @@ func (k *KV2) Writes() (dyna, perm int) { func (k *KV2) SetFilterBlocks(n uint64) (err error) { k.Mutex.Lock() defer k.Mutex.Unlock() - if err = k.PermKV.SetFilterBlocks(n); err != nil { + if err = k.perm().SetFilterBlocks(n); err != nil { return err } + if k.DynaKV == nil { + return nil // A heap has no window + } return k.DynaKV.SetFilterBlocks(n) } +// dynaLayer is what KV2 asks of its dynamic layer, whichever of the +// two it opened: sealed mutable segments (SegmentStore) or the heap +// with holes (HeapStore). +type dynaLayer interface { + Open() error + Close() error + Get(key [32]byte) ([]byte, error) + GetDeep(key [32]byte) ([]byte, error) + Put(key [32]byte, value []byte) error + AdvanceBlock(height uint64) + LiveRecords() uint64 + beginBlockSync() (blockSync, error) + compact(budget int64) (bool, error) // A heap moves at most budget bytes; a segment store ignores it + Stats() StoreStats +} + +// blockSync is the second half of a block sync, finished outside the +// shard's lock. +type blockSync interface{ finish() error } + +// permLayer is what KV2 asks of its permanent layer, whichever of the +// two it opened: sealed immutable segments (SegmentStore) or files of +// entries and buckets of keys (PermStore). +type permLayer interface { + Open() error + Close() error + Get(key [32]byte) ([]byte, error) + GetDeep(key [32]byte) ([]byte, error) + Put(key [32]byte, value []byte) error + PutIfAbsent(key [32]byte, value []byte) (existing []byte, existed bool, err error) + LiveCount() int + AdvanceBlock(height uint64) + SetFilterBlocks(n uint64) error + beginPermSeal(height uint64) (blockSync, error) + mergeBelow(height uint64) (bool, error) + Stats() StoreStats +} + +// perm is the permanent layer this shard opened. +func (k *KV2) perm() permLayer { + if k.Perm != nil { + return k.Perm + } + return k.PermKV +} + +// NewKV2Files is NewKV2 with both layers as files of entries: the +// dynamic layer a heap (heap.go), the permanent layer files with +// buckets of keys (perm.go). +func NewKV2Files(directory string, sealLimit uint64) (kv2 *KV2, err error) { + if kv2, err = NewKV2Heap(directory, sealLimit); err != nil { + return nil, err + } + if err = kv2.PermKV.Close(); err != nil { + return nil, err + } + os.RemoveAll(filepath.Join(directory, PermDirName)) + kv2.PermKV = nil + if kv2.Perm, err = NewPermStore(filepath.Join(directory, PermFilesDirName), MinFilterBlocks); err != nil { + return nil, err + } + return kv2, nil +} + +// dyna is the dynamic layer this shard opened. +func (k *KV2) dyna() dynaLayer { + if k.Heap != nil { + return k.Heap + } + return k.DynaKV +} + +// NewKV2Heap is NewKV2 with the dynamic layer as a heap with holes. +func NewKV2Heap(directory string, sealLimit uint64) (kv2 *KV2, err error) { + if kv2, err = NewKV2(directory, sealLimit); err != nil { + return nil, err + } + if err = kv2.DynaKV.Close(); err != nil { + return nil, err + } + os.RemoveAll(filepath.Join(directory, DynaDirName)) + kv2.DynaKV = nil + if kv2.Heap, err = NewHeapStore(filepath.Join(directory, HeapDirName)); err != nil { + return nil, err + } + return kv2, nil +} + func OpenKV2(directory string) (kv2 *KV2, err error) { kv2 = new(KV2) kv2.Directory = directory permDirName := filepath.Join(directory, PermDirName) // Add directory names dynaDirName := filepath.Join(directory, DynaDirName) // Add directory names - if kv2.PermKV, err = OpenSegmentStore(permDirName); err != nil { + // The permanent layer is whichever the store was built with + if _, statErr := os.Stat(filepath.Join(directory, PermFilesDirName, "perm.json")); statErr == nil { + if kv2.Perm, err = OpenPermStore(filepath.Join(directory, PermFilesDirName)); err != nil { + return nil, err + } + } else if kv2.PermKV, err = OpenSegmentStore(permDirName); err != nil { return nil, err } + // The dynamic layer is whichever the store was built with + if _, statErr := os.Stat(filepath.Join(directory, HeapDirName)); statErr == nil { + if kv2.Heap, err = OpenHeapStore(filepath.Join(directory, HeapDirName)); err != nil { + return nil, err + } + return kv2, nil + } if kv2.DynaKV, err = OpenSegmentStore(dynaDirName); err != nil { return nil, err } @@ -179,19 +286,21 @@ func (k *KV2) Open() error { } k.Mutex.Lock() defer k.Mutex.Unlock() - if err := k.PermKV.Open(); err != nil { + if err := k.perm().Open(); err != nil { return err } if k.SealLimit == 0 { // Restore what the database was built with; only a store // predating the persisted field falls back to a default - if limit := k.PermKV.SealLimit; limit > 0 { + if k.PermKV == nil { + k.SealLimit = int(DefaultBloomCapacity) + } else if limit := k.PermKV.SealLimit; limit > 0 { k.SealLimit = int(limit) } else { k.SealLimit = int(DefaultBloomCapacity) } } - if err := k.DynaKV.Open(); err != nil { + if err := k.dyna().Open(); err != nil { return err } k.opened.Store(true) @@ -212,8 +321,8 @@ func (k *KV2) Close() error { k.Mutex.Lock() defer k.Mutex.Unlock() k.opened.Store(false) - err := k.PermKV.Close() - if dynaErr := k.DynaKV.Close(); err == nil { + err := k.perm().Close() + if dynaErr := k.dyna().Close(); err == nil { err = dynaErr } return err @@ -225,7 +334,7 @@ func (k *KV2) GetDyna(key [32]byte) (value []byte, err error) { k.Mutex.RLock() defer k.Mutex.RUnlock() - if value, err = k.DynaKV.Get(key); err != nil { // Not in DynaKV, then return whatever + if value, err = k.dyna().Get(key); err != nil { // Not in DynaKV, then return whatever return nil, err } return value, nil @@ -244,7 +353,7 @@ func (k *KV2) GetPerm(key [32]byte) (value []byte, err error) { k.Mutex.RLock() defer k.Mutex.RUnlock() - if value, err = k.PermKV.Get(key); err != nil { // Not in PermKV, then return whatever + if value, err = k.perm().Get(key); err != nil { // Not in PermKV, then return whatever return nil, err } return value, nil @@ -258,7 +367,7 @@ func (k *KV2) GetPerm(key [32]byte) (value []byte, err error) { func (k *KV2) GetPermDeep(key [32]byte) (value []byte, err error) { k.Mutex.RLock() defer k.Mutex.RUnlock() - return k.PermKV.GetDeep(key) + return k.perm().GetDeep(key) } // Get @@ -284,7 +393,7 @@ func (k *KV2) Get(key [32]byte) (value []byte, err error) { defer k.Mutex.RUnlock() // Check and see if this is a key that has been changed - value, err = k.DynaKV.Get(key) + value, err = k.dyna().Get(key) switch { case err == nil: return value, nil @@ -297,7 +406,7 @@ func (k *KV2) Get(key [32]byte) (value []byte, err error) { // is how it says so. return nil, err } - return k.PermKV.Get(key) // Not in DynaKV; return whatever PermKV has + return k.perm().Get(key) // Not in DynaKV; return whatever PermKV has } // GetDeep @@ -308,14 +417,14 @@ func (k *KV2) GetDeep(key [32]byte) (value []byte, err error) { k.Mutex.RLock() defer k.Mutex.RUnlock() - value, err = k.DynaKV.GetDeep(key) + value, err = k.dyna().GetDeep(key) switch { case err == nil: return value, nil case !errors.Is(err, errNotFound): return nil, err // See Get: a failure is not an absence } - return k.PermKV.GetDeep(key) + return k.perm().GetDeep(key) } // PutDyna @@ -326,7 +435,7 @@ func (k *KV2) PutDyna(key [32]byte, value []byte) (writes int, err error) { k.Mutex.RLock() // Shared: see Put (issue #66) defer k.Mutex.RUnlock() k.dWrites.Add(1) - if err = k.DynaKV.Put(key, value); err != nil { + if err = k.dyna().Put(key, value); err != nil { return int(k.dWrites.Load()), err } autoSeal, err = k.sealDynaIfFull() @@ -341,7 +450,7 @@ func (k *KV2) PutPerm(key [32]byte, value []byte) (writes int, err error) { k.Mutex.RLock() // Shared: see Put (issue #66) defer k.Mutex.RUnlock() k.pWrites.Add(1) - if err = k.PermKV.Put(key, value); err != nil { + if err = k.perm().Put(key, value); err != nil { return int(k.pWrites.Load()), err } autoSeal, err = k.sealPermIfFull() @@ -374,7 +483,7 @@ func finishAutoSeal(p *pendingSeal, err error) error { // published and under the exclusive lock in Open, and the layer // serializes its own seal. func (k *KV2) sealPermIfFull() (p *pendingSeal, err error) { - if k.SealLimit <= 0 || k.PermKV.LiveCount() < k.SealLimit { + if k.Perm != nil || k.SealLimit <= 0 || k.PermKV.LiveCount() < k.SealLimit { return nil, nil } return k.PermKV.beginSealNext() @@ -388,8 +497,8 @@ func (k *KV2) sealPermIfFull() (p *pendingSeal, err error) { // replayed in full on every open. The caller must hold the KV2 lock, // shared or exclusive; see sealPermIfFull. func (k *KV2) sealDynaIfFull() (p *pendingSeal, err error) { - if k.SealLimit <= 0 || k.DynaKV.LiveRecords() < uint64(k.SealLimit) { - return nil, nil + if k.Heap != nil || k.SealLimit <= 0 || k.DynaKV.LiveRecords() < uint64(k.SealLimit) { + return nil, nil // A heap is never sealed: it has no tail to fill } return k.DynaKV.beginSealNext() } @@ -436,9 +545,9 @@ func (k *KV2) sealDynaIfFull() (p *pendingSeal, err error) { // out (issue #84; the two halves are seal.go's). func (k *KV2) Seal(height uint64) (meta SegmentMeta, err error) { k.Mutex.Lock() - perm, err := k.PermKV.beginSeal(height) - dyna, dynaErr := k.DynaKV.beginSync() - k.DynaKV.AdvanceBlock(height + 1) + perm, err := k.perm().beginPermSeal(height) + dyna, dynaErr := k.dyna().beginBlockSync() + k.dyna().AdvanceBlock(height + 1) k.Mutex.Unlock() // Both halves finish regardless of the other: a failure to sync @@ -452,7 +561,7 @@ func (k *KV2) Seal(height uint64) (meta SegmentMeta, err error) { }() } if perm != nil { - meta, err = perm.finish() + err = perm.finish() } wg.Wait() if err == nil { @@ -474,6 +583,10 @@ func (k *KV2) Seal(height uint64) (meta SegmentMeta, err error) { // KV2 lock here held every Put, Get and Seal of the shard for the // whole copy, which is the pause issue #57 measures. func (k *KV2) MergeBelow(height uint64) (meta SegmentMeta, merged bool, err error) { + if k.Perm != nil { + merged, err = k.Perm.mergeBelow(height) + return meta, merged, err + } return k.PermKV.MergeBelow(height) } @@ -517,12 +630,12 @@ func (k *KV2) Put(key [32]byte, value []byte) (writes int, err error) { k.Mutex.RLock() defer k.Mutex.RUnlock() - if value2, err2 := k.DynaKV.Get(key); err2 == nil { // Check. Is this a DynaKV key? + if value2, err2 := k.dyna().Get(key); err2 == nil { // Check. Is this a DynaKV key? if bytes.Equal(value, value2) { // If the key is in DynaKV, it stays there. return int(k.dWrites.Load()), nil // If the value is not changed, do nothing } k.dWrites.Add(1) - if err = k.DynaKV.Put(key, value); err != nil { // If the value DID change, update + if err = k.dyna().Put(key, value); err != nil { // If the value DID change, update return int(k.dWrites.Load()), err } autoSeal, err = k.sealDynaIfFull() @@ -534,7 +647,7 @@ func (k *KV2) Put(key [32]byte, value []byte) (writes int, err error) { // whether the key was there and then calling Put, which asked again // to enforce immutability, made a new key -- the common case, and a // miss by definition -- pay for the answer twice. - existing, existed, err := k.PermKV.PutIfAbsent(key, value) + existing, existed, err := k.perm().PutIfAbsent(key, value) if err != nil { return int(k.dWrites.Load()), err } @@ -553,7 +666,7 @@ func (k *KV2) Put(key [32]byte, value []byte) (writes int, err error) { return int(k.dWrites.Load()), nil } k.dWrites.Add(1) - if err = k.DynaKV.Put(key, value); err != nil { // If the perm value changed, it is now a DynaKV + if err = k.dyna().Put(key, value); err != nil { // If the perm value changed, it is now a DynaKV return int(k.dWrites.Load()), err } autoSeal, err = k.sealDynaIfFull() @@ -593,8 +706,11 @@ func (k *KV2) Put(key [32]byte, value []byte) (writes int, err error) { // weight rather than a wrong answer. // // TODO: Cleanse PermKV of keys in DynaKV -func (k *KV2) Compress() error { - if _, err := k.DynaKV.CompactHistory(); err != nil { +func (k *KV2) Compress() error { return k.compress(HeapCleanBytes) } + +// compress is Compress with the mover's budget for this call. +func (k *KV2) compress(budget int64) error { + if _, err := k.dyna().compact(budget); err != nil { return err } k.Mutex.Lock() diff --git a/database/kv_shard.go b/database/kv_shard.go index 9a31c74..504e237 100644 --- a/database/kv_shard.go +++ b/database/kv_shard.go @@ -82,6 +82,13 @@ type KVShard struct { // found" for data that is on disk. Shards []*KV2 + // Maintenance in rotation (Compress): the height last sealed, the + // height at the last call, and the shard next in line + maintMu sync.Mutex + sealed uint64 + maintAt uint64 + maintNext int + // Sets holds the finalized Perm data that has left the shards: one // block-set file per completed set of blocks, packed from every // shard's merged segment (blockset.go). Each shard's Perm layer @@ -103,6 +110,9 @@ func (k *KVShard) attachSets() (err error) { if shard == nil || shard.PermKV == nil { continue } + if shard.PermKV == nil { + continue // A file-backed permanent layer keeps its own deep history + } if err = shard.PermKV.attachCold(shardSets{k.Sets, i}); err != nil { return fmt.Errorf("shard %d: %w", i, err) } @@ -169,6 +179,7 @@ func OpenKVShard(directory string) (kVShard *KVShard, err error) { return nil, err } } + kVShard.phaseSnapshots() kVShard.useSharedBlockRecord() if err = kVShard.adoptBlockHeight(); err != nil { return nil, err @@ -195,6 +206,9 @@ func OpenKVShard(directory string) (kVShard *KVShard, err error) { // again, which commits and retires them if newest, ok := kVShard.Sets.Newest(); ok { for i, shard := range kVShard.Shards { + if shard.PermKV == nil { + continue + } if _, err = shard.PermKV.DropBelow(newest.Last + 1); err != nil { return nil, fmt.Errorf("shard %d: %w", i, err) } @@ -227,7 +241,34 @@ func NewKVShard(directory string, sealLimit uint64) (kvs *KVShard, err error) { // // How many to ask for is a question about rate, not about size; see // DefaultNumShards. +// NewKVShardHeapN is NewKVShardN with every shard's dynamic layer a +// heap with holes (heap.go). +func NewKVShardHeapN(directory string, shards int, sealLimit uint64) (kvs *KVShard, err error) { + return newKVShardN(directory, shards, sealLimit, NewKV2Heap) +} + +// NewKVShardFilesN is NewKVShardN with both layers as files of entries +// (heap.go, perm.go). +func NewKVShardFilesN(directory string, shards int, sealLimit uint64) (kvs *KVShard, err error) { + return newKVShardN(directory, shards, sealLimit, NewKV2Files) +} + func NewKVShardN(directory string, shards int, sealLimit uint64) (kvs *KVShard, err error) { + return newKVShardN(directory, shards, sealLimit, NewKV2) +} + +// phaseSnapshots spreads the heap shards' key-map snapshots over the +// snapshot period, so that a store's shards do not all write theirs +// on the same block. +func (k *KVShard) phaseSnapshots() { + for i, shard := range k.Shards { + if shard.Heap != nil { + shard.Heap.SetSnapshotPhase(uint64(i) * HeapSnapshotBlocks / uint64(len(k.Shards))) + } + } +} + +func newKVShardN(directory string, shards int, sealLimit uint64, newShard func(string, uint64) (*KV2, error)) (kvs *KVShard, err error) { if shards < 1 { return nil, fmt.Errorf("a database needs at least one shard, asked for %d", shards) } @@ -241,10 +282,11 @@ func NewKVShardN(directory string, shards int, sealLimit uint64) (kvs *KVShard, kvs.Shards = make([]*KV2, shards) for i := range kvs.Shards { // Then create all the shards shardDir := kvs.ShardDir(i) - if kvs.Shards[i], err = NewKV2(shardDir, sealLimit); err != nil { // Create the KV2 for each shard + if kvs.Shards[i], err = newShard(shardDir, sealLimit); err != nil { // Create the KV2 for each shard return nil, err } } + kvs.phaseSnapshots() kvs.useSharedBlockRecord() if kvs.Sets, err = NewSetStore(kvs.setDir()); err != nil { return nil, err @@ -446,6 +488,9 @@ func (k *KVShard) writeBlockHeight(height uint64) (err error) { // with the block it belongs to func (k *KVShard) adoptBlockHeight() error { height, err := k.readBlockHeight() + k.maintMu.Lock() + k.sealed = height + k.maintMu.Unlock() if err != nil { return err } @@ -457,10 +502,10 @@ func (k *KVShard) adoptBlockHeight() error { continue } if shard.PermKV != nil { - shard.PermKV.AdvanceBlock(height) + shard.perm().AdvanceBlock(height) } if shard.DynaKV != nil { // So that its window is where the set's is - shard.DynaKV.AdvanceBlock(height) + shard.dyna().AdvanceBlock(height) } } return nil @@ -471,7 +516,9 @@ func (k *KVShard) adoptBlockHeight() error { func (k *KVShard) useSharedBlockRecord() { for _, shard := range k.Shards { if shard != nil && shard.PermKV != nil { - shard.PermKV.ExternalBlockRecord = true + if shard.PermKV != nil { + shard.PermKV.ExternalBlockRecord = true + } } } } @@ -519,7 +566,13 @@ func (k *KVShard) SealBlock(height uint64) (err error) { return err } } - return k.writeBlockHeight(height + 1) + if err = k.writeBlockHeight(height + 1); err != nil { + return err + } + k.maintMu.Lock() + k.sealed = height + 1 + k.maintMu.Unlock() + return nil } // MergeFinalized @@ -592,6 +645,19 @@ func (k *KVShard) MergeFinalized(height uint64) (mergedShards int, err error) { // pack is done, and whichever of the two lands first, the drop that // follows removes what the set holds. func (k *KVShard) PackFinalized(height uint64) (meta SetMeta, packed bool, err error) { + // A file-backed permanent layer packs within the shard: its + // buckets retire into one run of keys, no bodies, no set file + if len(k.Shards) > 0 && k.Shards[0].Perm != nil { + for i, shard := range k.Shards { + if err = shard.Open(); err != nil { + return meta, false, fmt.Errorf("shard %d: %w", i, err) + } + if err = shard.Perm.Pack(); err != nil { + return meta, false, fmt.Errorf("shard %d: %w", i, err) + } + } + return meta, true, nil + } // One pack at a time. Every guard in here and below is a // check-then-act on state another pack can change: two calls read // the same watermark and pack the same segments into overlapping @@ -704,16 +770,22 @@ func (k *KVShard) Stats() (perm, dyna StoreStats) { dst.HistorySegments += s.HistorySegments dst.ActiveSegments += s.ActiveSegments dst.ResidentBloomBytes += s.ResidentBloomBytes + dst.ResidentIndexBytes += s.ResidentIndexBytes + dst.HeapFiles += s.HeapFiles + dst.HeapLiveBytes += s.HeapLiveBytes + dst.HeapDeadBytes += s.HeapDeadBytes + dst.HeapScannedBytes += s.HeapScannedBytes + dst.HeapMovedBytes += s.HeapMovedBytes } for _, shard := range k.Shards { if shard == nil { continue } if shard.PermKV != nil { - add(&perm, shard.PermKV.Stats()) + add(&perm, shard.perm().Stats()) } - if shard.DynaKV != nil { - add(&dyna, shard.DynaKV.Stats()) + if shard.Heap != nil || shard.DynaKV != nil { + add(&dyna, shard.dyna().Stats()) } } return perm, dyna @@ -722,11 +794,59 @@ func (k *KVShard) Stats() (perm, dyna StoreStats) { // Compress // Compress all the shards func (k *KVShard) Compress() (err error) { - for i, kvs := range k.Shards { - if err = kvs.Open(); err != nil { + if len(k.Shards) > 0 { + if err = k.Shards[0].Open(); err != nil { + return fmt.Errorf("shard 0: %w", err) + } + } + if len(k.Shards) == 0 || k.Shards[0].Heap == nil { + // Segment-store shards: a bounded pass on every shard, as before + for i, kvs := range k.Shards { + if err = kvs.Open(); err != nil { + return fmt.Errorf("shard %d: %w", i, err) + } + if err = kvs.Compress(); err != nil { + return fmt.Errorf("shard %d: %w", i, err) + } + } + return nil + } + // Heap shards: one slice of the store's maintenance, sized by the + // blocks since the last call. The store's mover rate is + // HeapStoreCleanBytes per HeapCleanPeriod blocks; a call takes the + // shards next in rotation, as many as the elapsed blocks earn, each + // with its share of what those blocks earned. Called every + // HeapCleanPeriod blocks it moves the whole budget over every + // shard; called every block it moves a little on the next shard, + // and the copies reach the device as a trickle rather than every + // shard's pass at once -- which, at nine stores in lockstep, was + // 300 MB and 72 barriers in one second, and queued the seals' + // fsyncs behind them for the next four. A hundred shards spread + // the same rate a hundred ways, and each call locks one. + k.maintMu.Lock() + n := len(k.Shards) + elapsed := k.sealed - k.maintAt + if k.maintAt == 0 || elapsed > HeapCleanPeriod { + elapsed = HeapCleanPeriod + } + k.maintAt = k.sealed + due := int(uint64(n) * elapsed / HeapCleanPeriod) + if due < 1 { + due = 1 + } + if due > n { + due = n + } + budget := HeapStoreCleanBytes * int64(elapsed) / int64(HeapCleanPeriod) / int64(due) + start := k.maintNext + k.maintNext = (start + due) % n + k.maintMu.Unlock() + for j := 0; j < due; j++ { + i := (start + j) % n + if err = k.Shards[i].Open(); err != nil { return fmt.Errorf("shard %d: %w", i, err) } - if err = kvs.Compress(); err != nil { + if err = k.Shards[i].compress(budget); err != nil { return fmt.Errorf("shard %d: %w", i, err) } } diff --git a/database/perm.go b/database/perm.go new file mode 100644 index 0000000..7831ebf --- /dev/null +++ b/database/perm.go @@ -0,0 +1,1324 @@ +package blockchainDB + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "sync" + "sync/atomic" +) + +// PermStore is the permanent layer as files of entries and buckets of +// keys (proposal docs/proposals/2026-09-16-entries-written-once.md, +// step 2). An entry is written once at the end of a data file and +// never moves; everything the store does to stay fast is done on +// 44-byte records that name entries (permindex.go). +// +// Files, in one directory: +// +// perm-N.dat entries, [len][height][key][value][crc] (heap.go's +// layout), appended in the order written, rolled at +// PermFileBytes; never rewritten +// runs-N.dat maintenance's run file: a bucket's run at every merge, +// a retired run at every pack; rolled at PermFileBytes +// and deleted once no run in it is referenced +// +// A block's delta -- its records' run, with its filter -- is appended +// to the block's data file right behind the block's entries, as an +// entry under the reserved key heap.go's deltas use, so a block is +// ONE fsync per shard for the permanent layer: entries and delta +// together. Open replays the deltas in the data files after the +// point the manifest records, verifies the last one's entries by +// their checksums before trusting it, and cuts the newest data file +// back to the last delta. +// +// perm.json the manifest: which runs are the window, the buckets +// and the retired history, and where in the run files +// the deltas not yet in the manifest begin +// +// Tiers, newest to oldest: +// +// - the live map: this block's records, in memory until the seal; +// - the window: the last FilterBlocks deltas, each a run with its +// filter resident. The protocol path reads the live map and the +// window and nothing older (spec 1.3): an immutable key the window +// does not hold is absent; +// - the buckets: PermBuckets buckets by the key's first byte, each a +// few sorted runs. A delta that leaves the window feeds each +// bucket's recent records; every bucket is merged every +// PermMergeEvery blocks, in rotation, its recent records written +// as a run and its runs folded by the store's ratio. The buckets +// hold only the history above the pack watermark, so a bucket is +// bounded by the pack period and hashed keys keep it even; +// - retired runs: at every pack, every bucket's runs and every +// delta below the watermark fold into one sorted run, and the +// buckets start again. Retired runs are the deep history. +// +// GetDeep walks the tiers in that order; Get stops after the window. +// +// Durability (spec 1.8): a seal fsyncs the data files the block wrote +// and then appends and fsyncs the block's delta, so a delta names +// only durable entries. Open reads the manifest, then every whole +// delta after the offset it records; a torn delta is dropped whole. +// Maintenance writes new runs, fsyncs them, and commits the manifest +// (written aside and renamed) before any run file is deleted: never +// unlink what a durable manifest names. +type PermStore struct { + Directory string + + mu sync.RWMutex + files map[uint32]*heapFile // Data files, by id + cur *heapFile // The data file the block appends to + nextID uint32 + dirty map[uint32]*heapFile + + runs map[uint32]*runFile // Run files, by id + maintRun *runFile // The run file maintenance appends to: never the seal's, so their barriers never share an inode + nextRun uint32 + live map[[32]byte]permRecord // This block's records + window []*permDelta // The last FilterBlocks deltas, oldest first + pending []*permDelta // Deltas below the window not yet in every bucket + buckets [PermBuckets]permBucket // The history above the watermark + retired []*permRun // The history below it, newest last + height uint64 + window_n uint64 // FilterBlocks + rotation int // The next bucket to merge + lastMerge uint64 // The height Merge last ran at: what decides how many buckets are due + + // The manifest's view: the run file and offset after which deltas + // are replayed on open + deltasFrom struct { + file uint32 + off int64 + } + syncMu sync.Mutex // Serializes seals with each other and with manifest commits + manifestAt uint64 // The height the manifest was last committed at + manifestDirty bool // Merges since then + closed bool + + putTotal, putDuplicate, lookups, windowHits, deepHits atomic.Uint64 + mergeRuns, foldRuns, packRuns atomic.Uint64 + indexBytes atomic.Uint64 +} + +// PermManifestBlocks is how many blocks may pass between manifest +// commits when no fold made one necessary. Counted in blocks so the +// caller's cadence does not set the commit cadence. +var PermManifestBlocks uint64 = 160 + +// PermBuckets is how many buckets a shard's history above the +// watermark is kept in, by the key's first byte. +const PermBuckets = 256 + +// PermMergeEvery is how many blocks pass between merges of one +// bucket: PermBuckets/PermMergeEvery buckets are merged each block. +// It bounds how many deltas wait unmerged, which every deep read +// probes and every merge reads a slice of: at 256, read p99 climbed +// from 7 to 47 us over five minutes and merge work from 157 to 305 s +// a minute. One window's worth keeps both flat; the extra folds +// cost a fraction of the index bytes. +var PermMergeEvery uint64 = MinFilterBlocks + +// PermFileBytes is the size data files and run files are rolled at. +var PermFileBytes int64 = 64 << 20 + +// PermFoldRatio is the ratio a bucket's runs fold by: a suffix of runs +// folds while each older run is no larger than 1/ratio of what has +// gathered behind it. +var PermFoldRatio = 0.25 + +type permDelta struct { + height uint64 + run *permRun + f *os.File // The data file the delta lies in + data uint32 // Its id +} + +type permBucket struct { + runs []*permRun // Oldest first + files []*runFile + merged uint64 // The height of the newest delta merged in +} + +// runFile is a file of runs and how many runs still reference it. +type runFile struct { + id uint32 + f *os.File + size int64 + refs int + unsynced bool // Runs written since the file was last fsynced +} + +func permDataName(id uint32) string { return fmt.Sprintf("perm-%06d.dat", id) } +func permRunName(id uint32) string { return fmt.Sprintf("runs-%06d.dat", id) } +func (rf *runFile) name() string { return permRunName(rf.id) } + +// NewPermStore creates an empty store in directory, replacing anything +// there. +func NewPermStore(directory string, filterBlocks uint64) (*PermStore, error) { + os.RemoveAll(directory) + if err := os.MkdirAll(directory, 0o755); err != nil { + return nil, err + } + p := &PermStore{Directory: directory, window_n: filterBlocks} + return p, p.Open() +} + +// OpenPermStore opens the store in directory as it was left. +func OpenPermStore(directory string) (*PermStore, error) { + if _, err := os.Stat(filepath.Join(directory, "perm.json")); err != nil { + return nil, fmt.Errorf("open perm at %s: %w", directory, err) + } + p := &PermStore{Directory: directory} + return p, p.Open() +} + +// permManifest is perm.json. +type permManifest struct { + Version int `json:"version"` + Height uint64 `json:"height"` + FilterBlocks uint64 `json:"filterBlocks"` + Rotation int `json:"rotation"` + NextData uint32 `json:"nextData"` + NextRun uint32 `json:"nextRun"` + DeltasFile uint32 `json:"deltasFile"` // Deltas after this file:offset are replayed + DeltasOff int64 `json:"deltasOff"` + Window []permRunRef `json:"window"` + Pending []permRunRef `json:"pending"` + Buckets []permBucketM `json:"buckets"` + Retired []permRunRef `json:"retired"` + DataFiles []uint32 `json:"dataFiles"` + RunFiles []uint32 `json:"runFiles"` +} + +type permRunRef struct { + File uint32 `json:"file"` // A data file for a delta, a run file otherwise + Off int64 `json:"off"` + Height uint64 `json:"height,omitempty"` +} + +type permBucketM struct { + Runs []permRunRef `json:"runs"` + Merged uint64 `json:"merged"` +} + +// Open loads the manifest and replays the deltas after it. +func (p *PermStore) Open() (err error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.files != nil { + return nil + } + p.files = map[uint32]*heapFile{} + p.dirty = map[uint32]*heapFile{} + p.runs = map[uint32]*runFile{} + p.live = map[[32]byte]permRecord{} + p.closed = false + var m permManifest + buf, err := os.ReadFile(filepath.Join(p.Directory, "perm.json")) + switch { + case errors.Is(err, os.ErrNotExist): + // A new store: the first data file and run file + if p.window_n == 0 { + p.window_n = MinFilterBlocks + } + if p.cur, err = p.newDataFile(); err != nil { + return err + } + return p.commitManifest() + case err != nil: + return err + } + if err = json.Unmarshal(buf, &m); err != nil { + return fmt.Errorf("perm.json: %w", err) + } + if m.Version != 1 { + return fmt.Errorf("perm.json: version %d, want 1", m.Version) + } + p.height, p.window_n, p.rotation, p.nextID, p.nextRun = m.Height, m.FilterBlocks, m.Rotation, m.NextData, m.NextRun + for _, id := range m.DataFiles { + f, err := os.OpenFile(filepath.Join(p.Directory, permDataName(id)), os.O_RDWR, 0o644) + if err != nil { + return fmt.Errorf("perm: the manifest names %s: %w", permDataName(id), err) + } + st, _ := f.Stat() + p.files[id] = &heapFile{id: id, f: f, size: st.Size()} + } + for _, id := range m.RunFiles { + f, err := os.OpenFile(filepath.Join(p.Directory, permRunName(id)), os.O_RDWR, 0o644) + if err != nil { + return fmt.Errorf("perm: the manifest names %s: %w", permRunName(id), err) + } + st, _ := f.Stat() + p.runs[id] = &runFile{id: id, f: f, size: st.Size()} + } + // Data files the seals rolled after the manifest's commit are not + // named by it, but they hold committed deltas: they are taken in + // id order from the manifest's next id for as long as they exist, + // and the replay below covers them. Run files past the manifest's + // next id are maintenance output that was never named; nothing + // durable refers to them, so they go, and their ids are free for + // O_EXCL creation again. + for { + f, err := os.OpenFile(filepath.Join(p.Directory, permDataName(p.nextID)), os.O_RDWR, 0o644) + if errors.Is(err, os.ErrNotExist) { + break + } + if err != nil { + return err + } + st, _ := f.Stat() + p.files[p.nextID] = &heapFile{id: p.nextID, f: f, size: st.Size()} + p.nextID++ + } + for id := p.nextRun; ; id++ { + err := os.Remove(filepath.Join(p.Directory, (&runFile{id: id}).name())) + if errors.Is(err, os.ErrNotExist) { + break + } + if err != nil { + return err + } + } + load := func(ref permRunRef, resident bool) (*permRun, *runFile, error) { + rf := p.runs[ref.File] + if rf == nil { + return nil, nil, fmt.Errorf("perm: run file %d not open", ref.File) + } + r, err := openPermRun(rf.f, ref.File, ref.Off, resident) + if err != nil { + return nil, nil, err + } + rf.refs++ + return r, rf, nil + } + loadDelta := func(ref permRunRef) (*permDelta, error) { + hf := p.files[ref.File] + if hf == nil { + return nil, fmt.Errorf("perm: data file %d not open", ref.File) + } + r, err := openPermRun(hf.f, ref.File, ref.Off, true) + if err != nil { + return nil, err + } + return &permDelta{height: ref.Height, run: r, f: hf.f, data: ref.File}, nil + } + for _, ref := range m.Window { + d, err := loadDelta(ref) + if err != nil { + return err + } + p.window = append(p.window, d) + } + for _, ref := range m.Pending { + d, err := loadDelta(ref) + if err != nil { + return err + } + p.pending = append(p.pending, d) + } + for i, bm := range m.Buckets { + if i >= PermBuckets { + break + } + p.buckets[i].merged = bm.Merged + for _, ref := range bm.Runs { + r, rf, err := load(ref, true) + if err != nil { + return err + } + p.buckets[i].runs = append(p.buckets[i].runs, r) + p.buckets[i].files = append(p.buckets[i].files, rf) + } + } + for _, ref := range m.Retired { + r, _, err := load(ref, false) + if err != nil { + return err + } + p.retired = append(p.retired, r) + } + // The current data file is the newest; the seal's run file is the + // manifest's; maintenance opens a file of its own when it runs + for _, hf := range p.files { + if p.cur == nil || hf.id > p.cur.id { + p.cur = hf + } + } + p.deltasFrom.file, p.deltasFrom.off = m.DeltasFile, m.DeltasOff + return p.replayDeltas() +} + +// replayDeltas applies every delta entry in the data files after the +// manifest's point, oldest first, into the window and pending. The +// last delta is trusted only if every entry it names checks -- one +// fsync covered it and its entries together -- and the newest data +// file is cut back to the last delta admitted. The caller holds the +// lock. +func (p *PermStore) replayDeltas() error { + ids := make([]uint32, 0, len(p.files)) + for id := range p.files { + if id >= p.deltasFrom.file { + ids = append(ids, id) + } + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + type found struct { + hf *heapFile + off, end int64 + run *permRun + } + var deltas []found + for _, id := range ids { + hf := p.files[id] + buf := make([]byte, hf.size) + if _, err := hf.f.ReadAt(buf, 0); err != nil && !errors.Is(err, io.EOF) { + return err + } + at := int64(0) + if id == p.deltasFrom.file { + at = p.deltasFrom.off + } + for at < int64(len(buf)) { + size, _, key, _, ok := decodeEntry(buf[at:]) + if size == 0 { + break + } + if ok && key == heapDeltaKey { + r, err := openPermRun(hf.f, id, at+heapHeader, true) + if err == nil { + deltas = append(deltas, found{hf: hf, off: at, end: at + size, run: r}) + } + } + at += size + } + } + var last *found + for i := range deltas { + d := &deltas[i] + if i == len(deltas)-1 && !p.deltaEntriesCheck(d.run, d.hf.f) { + break // The block did not commit + } + p.admit(&permDelta{height: d.run.height, run: d.run, f: d.hf.f, data: d.hf.id}) + if d.run.height >= p.height { + p.height = d.run.height + 1 + } + last = d + } + // The newest data file is cut back to what is committed: the last + // delta admitted, or the manifest's point + newest := p.files[ids[len(ids)-1]] + end := int64(0) + if newest.id == p.deltasFrom.file { + end = p.deltasFrom.off + } + if last != nil && last.hf == newest { + end = last.end + } + if newest.size > end { + if err := newest.f.Truncate(end); err != nil { + return err + } + newest.size = end + } + p.cur = newest + return nil +} + +// deltaEntriesCheck reads every entry a delta names and checks it. +func (p *PermStore) deltaEntriesCheck(r *permRun, f *os.File) bool { + recs, err := r.records(f) + if err != nil { + return false + } + for _, rec := range recs { + if _, err := p.readEntry(rec, rec.key); err != nil { + return false + } + } + return true +} + +// admit puts a delta in the window and moves what falls out of it to +// pending. The caller holds the lock. +func (p *PermStore) admit(d *permDelta) { + p.window = append(p.window, d) + for len(p.window) > int(p.window_n) { + p.pending = append(p.pending, p.window[0]) + p.window = p.window[1:] + } +} + +func (p *PermStore) newDataFile() (*heapFile, error) { + id := p.nextID + p.nextID++ + f, err := os.OpenFile(filepath.Join(p.Directory, permDataName(id)), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return nil, err + } + hf := &heapFile{id: id, f: f} + p.files[id] = hf + return hf, nil +} + +func (p *PermStore) newRunFile() (*runFile, error) { + id := p.nextRun + p.nextRun++ + rf := &runFile{id: id} + f, err := os.OpenFile(filepath.Join(p.Directory, rf.name()), os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return nil, err + } + rf.f = f + p.runs[id] = rf + return rf, nil +} + +// Close seals what is pending and closes the files. +func (p *PermStore) Close() error { + sync, err := p.beginSeal(p.height) + if err != nil { + return err + } + if err = sync.finish(); err != nil { + return err + } + p.mu.Lock() + defer p.mu.Unlock() + if p.manifestDirty { + if err = p.syncRunFiles(); err != nil { + return err + } + if err = p.commitManifest(); err != nil { + return err + } + if err = p.dropUnreferencedRunFiles(); err != nil { + return err + } + } + p.closed = true + for _, hf := range p.files { + if cerr := hf.f.Close(); err == nil { + err = cerr + } + } + for _, rf := range p.runs { + if cerr := rf.f.Close(); err == nil { + err = cerr + } + } + p.files, p.runs, p.cur, p.maintRun = nil, nil, nil, nil + return err +} + +// PutIfAbsent writes value under key unless the window already holds +// the key, in which case the existing value is returned instead. The +// window is the store's immutability horizon (spec 1.3). +func (p *PermStore) PutIfAbsent(key [32]byte, value []byte) (existing []byte, existed bool, err error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.closed { + return nil, false, errStoreClosed + } + p.putTotal.Add(1) + if rec, ok := p.live[key]; ok { + p.putDuplicate.Add(1) + v, err := p.readEntry(rec, key) + return v, true, err + } + for i := len(p.window) - 1; i >= 0; i-- { + d := p.window[i] + rec, found, err := d.run.lookup(d.f, key) + if err != nil { + return nil, false, err + } + if found { + p.putDuplicate.Add(1) + v, err := p.readEntry(rec, key) + return v, true, err + } + } + size := entrySize(len(value)) + if p.cur.size+size > PermFileBytes { + if p.cur, err = p.newDataFile(); err != nil { + return nil, false, err + } + } + rec := permRecord{key: key, file: p.cur.id, off: uint32(p.cur.size), n: uint32(len(value))} + if _, err = p.cur.f.WriteAt(encodeEntry(p.height, key, value), p.cur.size); err != nil { + return nil, false, err + } + p.cur.size += size + p.dirty[p.cur.id] = p.cur + p.live[key] = rec + return nil, false, nil +} + +// Put is PutIfAbsent for callers that only need the error; a key +// already held is not an error, the value stands. +func (p *PermStore) Put(key [32]byte, value []byte) error { + _, _, err := p.PutIfAbsent(key, value) + return err +} + +func (p *PermStore) readEntry(rec permRecord, key [32]byte) ([]byte, error) { + hf := p.files[rec.file] + if hf == nil { + return nil, fmt.Errorf("perm: data file %d not open", rec.file) + } + buf := make([]byte, heapHeader+int(rec.n)+heapTrailer) + if _, err := hf.f.ReadAt(buf, int64(rec.off)); err != nil { + return nil, err + } + return heapEntryValue(buf, key) +} + +// Get answers from the live map and the window; a key older than the +// window is absent here (spec 1.3), GetDeep reaches it. +func (p *PermStore) Get(key [32]byte) ([]byte, error) { + p.mu.RLock() + defer p.mu.RUnlock() + if p.closed { + return nil, errStoreClosed + } + p.lookups.Add(1) + if rec, ok := p.live[key]; ok { + p.windowHits.Add(1) + return p.readEntry(rec, key) + } + for i := len(p.window) - 1; i >= 0; i-- { + d := p.window[i] + rec, found, err := d.run.lookup(d.f, key) + if err != nil { + return nil, err + } + if found { + p.windowHits.Add(1) + return p.readEntry(rec, key) + } + } + return nil, errNotFound +} + +// GetDeep is Get, then the deltas not yet merged, the key's bucket +// newest run first, and the retired runs newest first. +func (p *PermStore) GetDeep(key [32]byte) ([]byte, error) { + v, err := p.Get(key) + if err == nil || !errors.Is(err, errNotFound) { + return v, err + } + p.mu.RLock() + defer p.mu.RUnlock() + for i := len(p.pending) - 1; i >= 0; i-- { + d := p.pending[i] + rec, found, err := d.run.lookup(d.f, key) + if err != nil { + return nil, err + } + if found { + p.deepHits.Add(1) + return p.readEntry(rec, key) + } + } + b := &p.buckets[key[0]] + for i := len(b.runs) - 1; i >= 0; i-- { + rec, found, err := b.runs[i].lookup(b.files[i].f, key) + if err != nil { + return nil, err + } + if found { + p.deepHits.Add(1) + return p.readEntry(rec, key) + } + } + for i := len(p.retired) - 1; i >= 0; i-- { + r := p.retired[i] + rf := p.runs[r.file] + rec, found, err := r.lookup(rf.f, key) + if err != nil { + return nil, err + } + if found { + p.deepHits.Add(1) + return p.readEntry(rec, key) + } + } + return nil, errNotFound +} + +// permSeal is a seal in flight: the data files to fsync, the delta +// to append. Holds syncMu until finished. +type permSeal struct { + p *PermStore + dirty []*heapFile + recs []permRecord + height uint64 + run *permRun // The delta as written, admitted once durable + hf *heapFile // The data file it was written to +} + +// beginSeal takes the block's records under the lock; finish makes +// them durable outside it. +func (p *PermStore) beginSeal(height uint64) (*permSeal, error) { + p.syncMu.Lock() + p.mu.Lock() + defer p.mu.Unlock() + if p.closed || p.files == nil { + p.syncMu.Unlock() + return nil, errStoreClosed + } + s := &permSeal{p: p, height: height} + for _, hf := range p.dirty { + s.dirty = append(s.dirty, hf) + } + p.dirty = map[uint32]*heapFile{} + s.recs = make([]permRecord, 0, len(p.live)) + for _, rec := range p.live { + s.recs = append(s.recs, rec) + } + p.live = map[[32]byte]permRecord{} // A put from here on is the next block's + sortPermRecords(s.recs) + if len(s.recs) == 0 { + return s, nil + } + // The delta's run goes into the block's data file behind the + // block's entries, under the reserved key: one fsync of that file + // covers both, and replay trusts the last delta only if every + // entry it names checks (spec 1.7) + var w bufWriterAt + run, err := writePermRun(&w, 0, p.cur.id, s.recs, s.height) + if err != nil { + p.syncMu.Unlock() + return nil, err + } + entry := encodeEntry(s.height, heapDeltaKey, w.buf) + if p.cur.size+int64(len(entry)) > PermFileBytes { + if p.cur, err = p.newDataFile(); err != nil { + p.syncMu.Unlock() + return nil, err + } + } + hf, at := p.cur, p.cur.size + if _, err = hf.f.WriteAt(entry, at); err != nil { + p.syncMu.Unlock() + return nil, err + } + hf.size += int64(len(entry)) + run.file, run.off, run.bloomAt = hf.id, at+heapHeader, at+heapHeader+run.bloomAt + s.run, s.hf = run, hf + if _, dirty := p.dirty[hf.id]; !dirty { + for _, d := range s.dirty { + if d == hf { + dirty = true + } + } + if !dirty { + s.dirty = append(s.dirty, hf) + } + } + return s, nil +} + +// finish: the block's entries and its delta durable, one fsync per +// file touched, with the lock released; then the delta admitted to +// the window. Releases syncMu. +func (s *permSeal) finish() error { + p := s.p + defer p.syncMu.Unlock() + for _, hf := range s.dirty { + if err := fsync(hf.f); err != nil { + return err + } + } + p.mu.Lock() + defer p.mu.Unlock() + if s.run != nil { + p.indexBytes.Add(uint64(s.run.bytes)) + p.admit(&permDelta{height: s.height, run: s.run, f: s.hf.f, data: s.hf.id}) + } + if s.height >= p.height { + p.height = s.height + 1 + } + return nil +} + +// bufWriterAt collects what a run writer writes at offset 0. +type bufWriterAt struct{ buf []byte } + +func (w *bufWriterAt) WriteAt(b []byte, off int64) (int, error) { + if need := int(off) + len(b); need > len(w.buf) { + w.buf = append(w.buf, make([]byte, need-len(w.buf))...) + } + copy(w.buf[off:], b) + return len(b), nil +} + +// AdvanceBlock sets the block new writes belong to. +func (p *PermStore) AdvanceBlock(height uint64) { + p.mu.Lock() + p.height = height + p.mu.Unlock() +} + +// BlockHeight is the block being written. +func (p *PermStore) BlockHeight() uint64 { + p.mu.RLock() + defer p.mu.RUnlock() + return p.height +} + +// LiveCount is the records this block has written so far. +func (p *PermStore) LiveCount() int { + p.mu.RLock() + defer p.mu.RUnlock() + return len(p.live) +} + +// maintFile is the run file maintenance appends to, rolled at +// PermFileBytes. The caller holds the lock. +func (p *PermStore) maintFile() (*runFile, error) { + if p.maintRun == nil || p.maintRun.size > PermFileBytes { + rf, err := p.newRunFile() + if err != nil { + return nil, err + } + p.maintRun = rf + } + return p.maintRun, nil +} + +// Merge is the maintenance step: the buckets due in rotation take +// their records from the pending deltas, fold, and the manifest is +// committed; deltas every bucket has absorbed are released. Runs +// are written and fsynced outside the lock; the lock is held to +// choose and to swap. +func (p *PermStore) Merge() error { + p.mu.Lock() + if p.closed { + p.mu.Unlock() + return errStoreClosed + } + if len(p.pending) == 0 { + p.mu.Unlock() + return nil + } + // Which buckets: every bucket is due once per PermMergeEvery + // blocks, so a call that comes after n blocks takes n/PermMergeEvery + // of them, whatever the caller's cadence + elapsed := p.height - p.lastMerge + if p.lastMerge == 0 || elapsed > PermMergeEvery { + elapsed = PermMergeEvery + } + due := int(uint64(PermBuckets) * elapsed / PermMergeEvery) + if due < 1 { + due = 1 + } + if due > PermBuckets { + due = PermBuckets + } + p.lastMerge = p.height + newest := p.pending[len(p.pending)-1].height + type job struct{ b int } + var jobs []job + for i := 0; i < due; i++ { + jobs = append(jobs, job{b: (p.rotation + i) % PermBuckets}) + } + rotation := (p.rotation + due) % PermBuckets + // Read the deltas' records for these buckets outside the lock: + // runs are immutable. The due buckets are a run of the rotation, + // so their records are one or two slices of each sorted delta + oldest := ^uint64(0) + for _, j := range jobs { + if p.buckets[j.b].merged < oldest { + oldest = p.buckets[j.b].merged + } + } + var ranges [][2]byte + if last := p.rotation + due - 1; last < PermBuckets { + ranges = append(ranges, [2]byte{byte(p.rotation), byte(last)}) + } else { + ranges = append(ranges, [2]byte{byte(p.rotation), PermBuckets - 1}, [2]byte{0, byte(last - PermBuckets)}) + } + pendingCopy := append([]*permDelta(nil), p.pending...) + p.mu.Unlock() + byBucket := map[int][][]permRecord{} + for _, d := range pendingCopy { + if d.height <= oldest { + continue // Every due bucket has this delta already + } + var recs []permRecord + for _, rg := range ranges { + part, err := d.run.recordsBucketRange(d.f, rg[0], rg[1]) + if err != nil { + return err + } + recs = append(recs, part...) + } + for _, j := range jobs { + if d.height <= p.buckets[j.b].merged { + continue + } + var mine []permRecord + for _, r := range recs { + if int(r.key[0]) == j.b { + mine = append(mine, r) + } + } + byBucket[j.b] = append(byBucket[j.b], mine) + } + } + // Write each due bucket's new run to the maintenance run file + p.mu.Lock() + rf, err := p.maintFile() + p.mu.Unlock() + if err != nil { + return err + } + var written []struct { + b int + run *permRun + } + for _, j := range jobs { + recs := mergePermRuns(byBucket[j.b]) + if len(recs) == 0 { + continue + } + p.mu.Lock() + at := rf.size + rf.size += 0 + p.mu.Unlock() + run, err := writePermRun(rf.f, at, rf.id, recs, 0) + if err != nil { + return err + } + p.mu.Lock() + rf.size = at + int64(run.bytes) + p.mu.Unlock() + p.indexBytes.Add(uint64(run.bytes)) + p.mergeRuns.Add(1) + written = append(written, struct { + b int + run *permRun + }{b: j.b, run: run}) + } + // No barrier here: a bucket's run is named by the manifest alone, + // so it needs to be durable before the manifest commit and not + // before. Every merge call once ended with an fsync of its run + // file, which at a call per shard per block was 72-144 barriers + // a second at nine stores and doubled every seal's fsync. + // Swap: the new runs join their buckets; then fold what the ratio + // says, one bucket at a time + p.mu.Lock() + rf.unsynced = true + for _, w := range written { + bk := &p.buckets[w.b] + bk.runs = append(bk.runs, w.run) + bk.files = append(bk.files, rf) + rf.refs++ + } + for _, j := range jobs { + p.buckets[j.b].merged = newest + } + p.rotation = rotation + // Deltas every bucket has absorbed are released + minMerged := ^uint64(0) + for i := range p.buckets { + if p.buckets[i].merged < minMerged { + minMerged = p.buckets[i].merged + } + } + var keep []*permDelta + for _, d := range p.pending { + if d.height > minMerged { + keep = append(keep, d) // A delta in a data file costs nothing to drop: the file stays + } + } + p.pending = keep + folds := p.planFolds() + p.mu.Unlock() + // The folds' runs are written without a barrier too; the manifest + // commit syncs every run file written since the last + for _, f := range folds { + if _, err := p.fold(f); err != nil { + return err + } + } + // The manifest names the new runs. It is committed when a fold + // left run files to drop, and otherwise every PermManifestEvery + // merges: the runs are durable in their file already, and a + // merge a crash loses is done again from the pending deltas, so a + // commit per merge -- two barriers a shard -- bought nothing but + // a shorter replay. Open removes the run files a lost merge left + // unnamed. + p.mu.Lock() + defer p.mu.Unlock() + p.manifestDirty = true + // A commit is due by age, or when a fold left a run file that + // nothing references, since only a commit can drop it. A fold + // alone is not a reason: with a merge every block a bucket folds + // on nearly every call, and a commit each time was two barriers + // a shard a block. + droppable := false + for _, rf := range p.runs { + if rf.refs == 0 && rf != p.maintRun { + droppable = true + break + } + } + if !droppable && p.height-p.manifestAt < PermManifestBlocks { + return nil + } + if err := p.syncRunFiles(); err != nil { + return err + } + if err := p.commitManifest(); err != nil { + return err + } + return p.dropUnreferencedRunFiles() +} + +type permFold struct { + b int + at int // The runs from this index on fold into one + count int +} + +// planFolds chooses, per bucket, the suffix of runs the ratio says to +// fold. The caller holds the lock. +func (p *PermStore) planFolds() (folds []permFold) { + for b := range p.buckets { + runs := p.buckets[b].runs + if len(runs) < 2 { + continue + } + var behind uint32 + i := len(runs) - 1 + for ; i >= 0; i-- { + if i < len(runs)-1 && float64(runs[i].count)*PermFoldRatio > float64(behind) { + break + } + behind += runs[i].count + } + if n := len(runs) - (i + 1); n >= 2 { + folds = append(folds, permFold{b: b, at: i + 1, count: n}) + } + } + return folds +} + +// fold merges a bucket's chosen runs into one, written to the +// maintenance run file (returned, for the caller to sync), and swaps +// it in under the lock. +func (p *PermStore) fold(f permFold) (*runFile, error) { + p.mu.RLock() + bk := &p.buckets[f.b] + if f.at+f.count > len(bk.runs) { + p.mu.RUnlock() + return nil, nil // The bucket changed under us; next time + } + runs := append([]*permRun(nil), bk.runs[f.at:f.at+f.count]...) + files := append([]*runFile(nil), bk.files[f.at:f.at+f.count]...) + p.mu.RUnlock() + inputs := make([][]permRecord, len(runs)) + for i, r := range runs { + recs, err := r.records(files[i].f) + if err != nil { + return nil, err + } + inputs[i] = recs + } + merged := mergePermRuns(inputs) + p.mu.Lock() + rf, err := p.maintFile() + if err != nil { + p.mu.Unlock() + return nil, err + } + at := rf.size + rf.size += 0 + p.mu.Unlock() + run, err := writePermRun(rf.f, at, rf.id, merged, 0) + if err != nil { + return nil, err + } + p.indexBytes.Add(uint64(run.bytes)) + p.foldRuns.Add(1) + p.mu.Lock() + defer p.mu.Unlock() + rf.size = at + int64(run.bytes) + rf.unsynced = true + bk = &p.buckets[f.b] + if f.at+f.count > len(bk.runs) { + return rf, nil + } + for _, old := range files { + old.refs-- + } + rest := append([]*permRun(nil), bk.runs[f.at+f.count:]...) + restF := append([]*runFile(nil), bk.files[f.at+f.count:]...) + bk.runs = append(append(bk.runs[:f.at], run), rest...) + bk.files = append(append(bk.files[:f.at], rf), restF...) + rf.refs++ + return rf, nil +} + +// Pack retires the history above the watermark: every bucket's runs +// and every pending delta fold into one sorted run, and the buckets +// start again. The retired run is the deep history's newest. +func (p *PermStore) Pack() error { + p.mu.RLock() + var inputs [][]permRecord + var release []*runFile + for _, d := range p.pending { + recs, err := d.run.records(d.f) + if err != nil { + p.mu.RUnlock() + return err + } + inputs = append(inputs, recs) + } + for b := range p.buckets { + for i, r := range p.buckets[b].runs { + recs, err := r.records(p.buckets[b].files[i].f) + if err != nil { + p.mu.RUnlock() + return err + } + inputs = append(inputs, recs) + release = append(release, p.buckets[b].files[i]) + } + } + height := p.height + p.mu.RUnlock() + if len(inputs) == 0 { + return nil + } + merged := mergePermRuns(inputs) + p.mu.Lock() + rf, err := p.maintFile() + if err != nil { + p.mu.Unlock() + return err + } + at := rf.size + p.mu.Unlock() + run, err := writePermRun(rf.f, at, rf.id, merged, height) + if err != nil { + return err + } + if err = fsync(rf.f); err != nil { + return err + } + p.indexBytes.Add(uint64(run.bytes)) + p.packRuns.Add(1) + p.mu.Lock() + defer p.mu.Unlock() + rf.size = at + int64(run.bytes) + run.bloom = nil // Retired runs are probed cold + p.retired = append(p.retired, run) + rf.refs++ + for _, r := range release { + r.refs-- + } + p.pending = nil + for b := range p.buckets { + p.buckets[b] = permBucket{merged: height} + } + if err := p.syncRunFiles(); err != nil { + return err + } + if err := p.commitManifest(); err != nil { + return err + } + return p.dropUnreferencedRunFiles() +} + +// syncRunFiles makes every run file written since its last fsync +// durable, with the lock released for the barriers. Maintenance is +// one pass at a time, so nothing writes a run file meanwhile. The +// caller holds the lock and gets it back. +func (p *PermStore) syncRunFiles() error { + var dirty []*runFile + for _, rf := range p.runs { + if rf.unsynced { + dirty = append(dirty, rf) + } + } + if len(dirty) == 0 { + return nil + } + p.mu.Unlock() + var err error + for _, rf := range dirty { + if err = fsync(rf.f); err != nil { + break + } + } + p.mu.Lock() + if err != nil { + return err + } + for _, rf := range dirty { + rf.unsynced = false + } + return nil +} + +// commitManifest writes perm.json: encoded under the lock, written, +// fsynced and renamed with the lock released, so that a merge's +// manifest commit -- every twenty blocks, per shard -- is not a +// barrier the shard's puts and seal wait behind (spec 1.6). A delta +// the seal appends meanwhile lies after the offset the manifest +// records, and open replays it. The caller holds the lock and gets +// it back. +func (p *PermStore) commitManifest() error { + buf, err := p.encodeManifest() + if err != nil { + return err + } + p.mu.Unlock() + err = p.writeManifest(buf) + p.mu.Lock() + if err == nil { + p.manifestAt, p.manifestDirty = p.height, false + } + return err +} + +// encodeManifest is the manifest as of now. The caller holds the lock. +func (p *PermStore) encodeManifest() ([]byte, error) { + m := permManifest{Version: 1, Height: p.height, FilterBlocks: p.window_n, Rotation: p.rotation, NextData: p.nextID, NextRun: p.nextRun} + ref := func(r *permRun, rf *runFile, height uint64) permRunRef { + return permRunRef{File: rf.id, Off: r.off, Height: height} + } + for _, d := range p.window { + m.Window = append(m.Window, permRunRef{File: d.data, Off: d.run.off, Height: d.height}) + } + for _, d := range p.pending { + m.Pending = append(m.Pending, permRunRef{File: d.data, Off: d.run.off, Height: d.height}) + } + for b := range p.buckets { + bm := permBucketM{Merged: p.buckets[b].merged} + for i, r := range p.buckets[b].runs { + bm.Runs = append(bm.Runs, ref(r, p.buckets[b].files[i], 0)) + } + m.Buckets = append(m.Buckets, bm) + } + for _, r := range p.retired { + m.Retired = append(m.Retired, permRunRef{File: r.file, Off: r.off, Height: r.height}) + } + for id := range p.files { + m.DataFiles = append(m.DataFiles, id) + } + // Only the run files kept: a file nothing references is dropped + // once this manifest is durable, and a durable manifest must never + // name a file that is gone (1.7, 1.8) + for id, rf := range p.runs { + if rf.refs > 0 || rf == p.maintRun { + m.RunFiles = append(m.RunFiles, id) + } + } + sort.Slice(m.DataFiles, func(i, j int) bool { return m.DataFiles[i] < m.DataFiles[j] }) + sort.Slice(m.RunFiles, func(i, j int) bool { return m.RunFiles[i] < m.RunFiles[j] }) + // Deltas sealed after this point lie in the data files from the + // current one's end; open replays from there + m.DeltasFile, m.DeltasOff = p.cur.id, p.cur.size + p.deltasFrom.file, p.deltasFrom.off = m.DeltasFile, m.DeltasOff + return json.MarshalIndent(m, "", " ") +} + +// writeManifest writes the encoded manifest aside, fsyncs it, renames +// it into place and fsyncs the directory. No lock is held. +func (p *PermStore) writeManifest(buf []byte) error { + path := filepath.Join(p.Directory, "perm.json") + tmp := path + segTmpSuffix + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return err + } + if _, err = f.Write(buf); err != nil { + f.Close() + return err + } + if err = fsync(f); err != nil { + f.Close() + return err + } + if err = f.Close(); err != nil { + return err + } + if err = os.Rename(tmp, path); err != nil { + return err + } + return fsyncDir(p.Directory) +} + +// dropUnreferencedRunFiles deletes run files no run references, after +// the manifest that no longer names them is durable. The caller +// holds the lock. +func (p *PermStore) dropUnreferencedRunFiles() error { + for id, rf := range p.runs { + if rf.refs > 0 || rf == p.maintRun { + continue + } + rf.f.Close() + delete(p.runs, id) + if err := os.Remove(filepath.Join(p.Directory, rf.name())); err != nil { + return err + } + } + return nil +} + +// Counters reports the maintenance so far: merge runs, folds, packs, +// and the index bytes written. +func (p *PermStore) Counters() (merges, folds, packs, indexBytes uint64) { + return p.mergeRuns.Load(), p.foldRuns.Load(), p.packRuns.Load(), p.indexBytes.Load() +} + +// Stats is the store's report. +func (p *PermStore) Stats() StoreStats { + p.mu.RLock() + defer p.mu.RUnlock() + var runs int + for b := range p.buckets { + runs += len(p.buckets[b].runs) + } + return StoreStats{ + PutTotal: p.putTotal.Load(), + PutNew: p.putTotal.Load() - p.putDuplicate.Load(), + PutDuplicate: p.putDuplicate.Load(), + LookupTotal: p.lookups.Load(), + LiveHit: p.windowHits.Load(), + ActiveSegments: len(p.window), + HistorySegments: runs + len(p.retired), + HeapFiles: len(p.files) + len(p.runs), + HeapMovedBytes: p.indexBytes.Load(), + } +} + +// beginPermSeal and mergeBelow are the permLayer surface (kv_2.go). +func (p *PermStore) beginPermSeal(height uint64) (blockSync, error) { + s, err := p.beginSeal(height) + if err != nil { + return nil, err + } + return s, nil +} + +func (p *PermStore) mergeBelow(uint64) (bool, error) { return true, p.Merge() } + +// SetFilterBlocks sets the window. +func (p *PermStore) SetFilterBlocks(n uint64) error { + if n < MinFilterBlocks { + return fmt.Errorf("perm: filter blocks %d below the minimum %d", n, MinFilterBlocks) + } + p.mu.Lock() + p.window_n = n + p.mu.Unlock() + return nil +} diff --git a/database/perm_roll_test.go b/database/perm_roll_test.go new file mode 100644 index 0000000..832ee06 --- /dev/null +++ b/database/perm_roll_test.go @@ -0,0 +1,116 @@ +package blockchainDB + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// A data file rolled by a seal between two manifest commits is still +// replayed on reopen: every block sealed before Close is found, and +// the store keeps writing after (spec 1.7: what a seal made durable +// stays durable). +func TestPermStoreReopenAcrossRolledFiles(t *testing.T) { + size := PermFileBytes + PermFileBytes = 64 << 10 + defer func() { PermFileBytes = size }() + dir := filepath.Join(t.TempDir(), "perm") + p, err := NewPermStore(dir, MinFilterBlocks) + require.NoError(t, err) + fr := NewFastRandom([]byte{7}) + var keys [][32]byte + value := make([]byte, 1000) + for b := uint64(1); b <= 8; b++ { // 8 blocks x 40 KB: several rolls, no merge + p.AdvanceBlock(b) + for i := 0; i < 40; i++ { + k := fr.NextHash() + keys = append(keys, k) + value[0] = byte(b) + require.NoError(t, p.Put(k, value)) + } + sealPerm(t, p, b) + } + require.Greater(t, len(p.files), 1, "the test must roll a data file") + require.NoError(t, p.Close()) + + p, err = OpenPermStore(dir) + require.NoError(t, err) + for i, k := range keys { + v, err := p.GetDeep(k) + require.NoError(t, err, "key %d after reopen", i) + require.Equal(t, byte(i/40+1), v[0]) + } + require.Equal(t, uint64(9), p.BlockHeight()) + // And the store keeps going: a new block, sealed, merged, reopened + p.AdvanceBlock(9) + k := fr.NextHash() + require.NoError(t, p.Put(k, value)) + sealPerm(t, p, 9) + require.NoError(t, p.Merge()) + require.NoError(t, p.Close()) + p, err = OpenPermStore(dir) + require.NoError(t, err) + _, err = p.GetDeep(k) + require.NoError(t, err) + _, err = p.GetDeep(keys[0]) + require.NoError(t, err) + require.NoError(t, p.Close()) +} + +// Merges between manifest commits survive a crash: the store reopened +// from the stale manifest finds every key, does the lost merges again +// from the pending deltas, and creates run files without colliding +// with the ones the lost merges left behind. +func TestPermStoreReopenWithoutManifestCommit(t *testing.T) { + every, size := PermMergeEvery, PermFileBytes + PermMergeEvery, PermFileBytes = 4, 64<<10 + defer func() { PermMergeEvery, PermFileBytes = every, size }() + dir := filepath.Join(t.TempDir(), "perm") + p, err := NewPermStore(dir, MinFilterBlocks) + require.NoError(t, err) + fr := NewFastRandom([]byte{9}) + var keys [][32]byte + value := make([]byte, 500) + for b := uint64(1); b <= 3*MinFilterBlocks; b++ { + p.AdvanceBlock(b) + for i := 0; i < 50; i++ { + k := fr.NextHash() + keys = append(keys, k) + require.NoError(t, p.Put(k, value)) + } + sealPerm(t, p, b) + if b%3 == 0 { + require.NoError(t, p.Merge()) + } + } + // Keep going until a merge is left uncommitted (a fold commits) + last := uint64(3 * MinFilterBlocks) + for !p.manifestDirty { + last++ + require.Less(t, last, uint64(6*MinFilterBlocks), "a merge without a fold must come") + p.AdvanceBlock(last) + k := fr.NextHash() + keys = append(keys, k) + require.NoError(t, p.Put(k, value)) + sealPerm(t, p, last) + require.NoError(t, p.Merge()) + } + // No Close: the manifest on disk is the last commit's + q, err := OpenPermStore(dir) + require.NoError(t, err) + for i, k := range keys { + _, err := q.GetDeep(k) + require.NoError(t, err, "key %d after reopen", i) + } + q.AdvanceBlock(last + 1) + require.NoError(t, q.Put(fr.NextHash(), value)) + sealPerm(t, q, last+1) + require.NoError(t, q.Merge(), "a new run file must not collide with a lost merge's") + require.NoError(t, q.Close()) + q, err = OpenPermStore(dir) + require.NoError(t, err) + _, err = q.GetDeep(keys[0]) + require.NoError(t, err) + require.NoError(t, q.Close()) +} diff --git a/database/perm_sizing_sim_test.go b/database/perm_sizing_sim_test.go new file mode 100644 index 0000000..b8d99b5 --- /dev/null +++ b/database/perm_sizing_sim_test.go @@ -0,0 +1,278 @@ +package blockchainDB + +import ( + "flag" + "fmt" + "sort" + "strings" + "testing" +) + +var simFlag = flag.Bool("sim", false, "run the permanent-index sizing simulation (TestPermSizingSim)") + +// A sizing simulation for the permanent layer's bucketed long search +// (docs/proposals/2026-09-16-entries-written-once.md, step 2). Not a +// test of the store: it models the index maintenance the proposal +// describes and prints, for a sweep of bucket counts, what a shard +// pays per block and what a lookup walks. Run with +// +// go test ./database/ -run TestPermSizingSim -v -sim +// +// The model, per shard: every block appends K records and seals a +// delta of K sorted 44-byte index records with a filter. A delta +// leaving the window (N blocks) feeds each record to its bucket's +// resident recent section. Every block the next bucket in rotation +// writes its recent section as a new sorted run and folds its runs by +// the store's ratio rule (compactionRunWithin: a suffix of runs is +// folded while each older run is no larger than 1/ratio of what has +// gathered behind it). Today's cost, for comparison: every +// MergeEvery blocks the shard folds its finished segments by copying +// bodies (~V bytes a record) and rebuilding the index, under the same +// ratio rule over merged segments; every PackEvery blocks the merged +// segments are copied again into a set. +func TestPermSizingSim(t *testing.T) { + if !*simFlag { + t.Skip("a sizing simulation; run with -sim") + } + const ( + K = 1040 // Permanent records per block per shard (soak: 8.3k per store, 8 shards) + V = 300 // Bytes a record's body costs to copy + rec = 44 // Bytes an index record costs + N = 20 // The window, blocks + ratio = 0.25 + mergeEvery = 20 + packEvery = 1000 + bloomBits = 12 // Bits per key in a filter + ) + horizons := []int{86_400, 7 * 86_400, 30 * 86_400} + buckets := []int{1, 64, 256, 1024, 4096} + splits := []int64{32 << 20, 64 << 20} // Bucket index bytes at which a bucket splits in two + const ( + M = 256 // Every bucket is merged every M blocks: B/M buckets a block + residentBudget = 64 << 20 // Filter bytes resident per shard, newest runs first; the rest are probed cold + runFile = 64 << 20 // Runs are appended to run files of this size; a run is (file, offset, length) + ) + + var out strings.Builder + fmt.Fprintf(&out, "\nPermanent index, one shard, K=%d records/block, ratio %.2f, window %d blocks\n", K, ratio, N) + for _, blocks := range horizons { + fmt.Fprintf(&out, "\n== %d blocks (%.0f days), %d M records ==\n", blocks, float64(blocks)/86_400, K*blocks/1_000_000) + // Today + today := simToday(blocks, K, V, rec, ratio, mergeEvery, packEvery) + fmt.Fprintf(&out, "today: index+body bytes written %.1f GB (%.1f MB/block avg), largest pass %.0f MB, merged segments at end %d, sets %d\n", + float64(today.written)/1e9, float64(today.written)/float64(blocks)/1e6, float64(today.largest)/1e6, today.segments, today.sets) + fmt.Fprintf(&out, "%-22s %12s %11s %11s %11s %9s %11s %11s %9s\n", "buckets", "idx KB/blk", "largest MB", "runs/bkt", "probes", "files", "resident MB", "cold probes", "recent MB") + for _, b := range buckets { + r := simBuckets(blocks, K, rec, N, ratio, b, 0, M, bloomBits, residentBudget, runFile) + fmt.Fprintf(&out, "%-22d %12.0f %11.1f %11.1f %11.1f %9d %11.1f %11.1f %9.1f\n", + b, float64(r.written)/float64(blocks)/1e3, float64(r.largest)/1e6, r.runsAvg, r.probes, r.files, float64(r.resident)/1e6, r.coldProbes, float64(r.recent)/1e6) + } + for _, sp := range splits { + for _, rt := range []float64{ratio, 0.1} { + r := simBuckets(blocks, K, rec, N, rt, 256, sp, M, bloomBits, residentBudget, runFile) + fmt.Fprintf(&out, "%-22s %12.0f %11.1f %11.1f %11.1f %9d %11.1f %11.1f %9.1f -> %d buckets\n", + fmt.Sprintf("256 split@%dMB r=%.2f", sp>>20, rt), float64(r.written)/float64(blocks)/1e3, float64(r.largest)/1e6, r.runsAvg, r.probes, r.files, float64(r.resident)/1e6, r.coldProbes, float64(r.recent)/1e6, r.buckets) + } + } + } + fmt.Fprintf(&out, "\nidx KB/blk: index bytes written per block, runs and folds; largest: the biggest single fold (or split);\n") + fmt.Fprintf(&out, "runs/bkt: sorted runs a lookup below the window probes (one filter each); probes: for a key not in the\n") + fmt.Fprintf(&out, "shard, window filter + recent section + runs, of which cold probes are on disk (K byte reads each);\n") + fmt.Fprintf(&out, "files: run files of %d MB; resident: filters within a %d MB budget, newest runs first; recent: the\n", runFile>>20, residentBudget>>20) + fmt.Fprintf(&out, "recent sections in memory (every bucket merged every %d blocks); split@S: a bucket whose runs exceed S\n", M) + fmt.Fprintf(&out, "splits in two; r: the fold ratio (a run folds into the older while the older is no larger than 1/r of it).\n") + t.Log(out.String()) +} + +type simResult struct { + written, largest int64 + runsAvg, probes float64 + coldProbes float64 + files, buckets int + resident, recent int64 +} + +// simBuckets models the bucketed long search. split > 0 splits a +// bucket in two once its runs hold more than split bytes (each half +// keeps half of every run: the keys are hashed, so a split is a +// rewrite of the bucket, counted as a fold). B/M buckets are merged +// each block so every bucket is merged every M blocks. +func simBuckets(blocks, K, rec, N int, ratio float64, B int, split int64, M, bloomBits int, residentBudget, runFile int64) simResult { + type bucket struct { + recent int64 // Records in the resident recent section + runs []int64 // Records per run, oldest first + } + bs := make([]bucket, B) + var r simResult + rotation := 0 + fold := func(b *bucket) { + if len(b.runs) < 2 { + return + } + var behind int64 + i := len(b.runs) - 1 + for ; i >= 0; i-- { + if i < len(b.runs)-1 && float64(b.runs[i])*ratio > float64(behind) { + break + } + behind += b.runs[i] + } + run := b.runs[i+1:] + if len(run) >= 2 { + var total int64 + for _, n := range run { + total += n + } + r.written += total * int64(rec) + if total*int64(rec) > r.largest { + r.largest = total * int64(rec) + } + b.runs = append(b.runs[:i+1], total) + } + } + for blk := 0; blk < blocks; blk++ { + if blk >= N { + perBucket := int64(K) / int64(len(bs)) + extra := int64(K) - perBucket*int64(len(bs)) + for i := range bs { + bs[i].recent += perBucket + } + for i := int64(0); i < extra; i++ { + bs[(blk*7+int(i))%len(bs)].recent++ + } + } + visits := (len(bs) + M - 1) / M + for v := 0; v < visits; v++ { + b := &bs[rotation%len(bs)] + rotation++ + if split > 0 { + var total int64 + for _, n := range b.runs { + total += n + } + if total*int64(rec) > split { + r.written += total * int64(rec) + if total*int64(rec) > r.largest { + r.largest = total * int64(rec) + } + half := bucket{recent: b.recent / 2} + b.recent -= half.recent + for i, n := range b.runs { + b.runs[i] = n / 2 + half.runs = append(half.runs, n-n/2) + } + bs = append(bs, half) + b = &bs[(rotation-1)%len(bs)] + } + } + if b.recent == 0 { + continue + } + r.written += b.recent * int64(rec) + b.runs = append(b.runs, b.recent) + b.recent = 0 + fold(b) + } + } + // Filters: newest runs first across the shard, within the budget + var runs, cold int + type runRef struct{ n int64 } + var newest, older []int64 // Per bucket: the newest run's records, then the rest + for _, b := range bs { + runs += len(b.runs) + r.recent += b.recent * int64(rec) + for i, n := range b.runs { + if i == len(b.runs)-1 { + newest = append(newest, n) + } else { + older = append(older, n) + } + } + } + budget := residentBudget + for _, n := range append(newest, older...) { + bytes := n * int64(bloomBits) / 8 + if budget >= bytes { + budget -= bytes + r.resident += bytes + } else { + cold++ + } + } + r.resident += r.recent + var indexBytes int64 + for _, b := range bs { + for _, n := range b.runs { + indexBytes += n * int64(rec) + } + } + r.files = int((indexBytes + runFile - 1) / runFile) + r.buckets = len(bs) + r.runsAvg = float64(runs) / float64(len(bs)) + r.coldProbes = float64(cold) / float64(len(bs)) + r.probes = 2 + r.runsAvg + _ = runRef{} + return r +} + +type simTodayResult struct { + written, largest int64 + segments, sets int +} + +// simToday models the store as it is: segments folded by copying +// bodies every mergeEvery blocks under the ratio rule, and packed +// every packEvery blocks by copying again. +func simToday(blocks, K, V, rec int, ratio float64, mergeEvery, packEvery int) simTodayResult { + perRecord := int64(V + rec) + var segs []int64 // Records per merged segment, oldest first + var r simTodayResult + pending := 0 // Finished blocks not yet merged + for blk := 1; blk <= blocks; blk++ { + pending++ + if blk%mergeEvery == 0 { + // The finished blocks become one merged segment, copied + n := int64(pending * K) + r.written += n * perRecord + segs = append(segs, n) + pending = 0 + var behind int64 + i := len(segs) - 1 + for ; i >= 0; i-- { + if i < len(segs)-1 && float64(segs[i])*ratio > float64(behind) { + break + } + behind += segs[i] + } + run := segs[i+1:] + if len(run) >= 2 { + var total int64 + for _, n := range run { + total += n + } + r.written += total * perRecord + if total*perRecord > r.largest { + r.largest = total * perRecord + } + segs = append(segs[:i+1], total) + } + } + if blk%packEvery == 0 { + // Everything merged so far below the watermark is copied + // into a set and dropped from the shard + var total int64 + for _, n := range segs { + total += n + } + r.written += total * perRecord + if total*perRecord > r.largest { + r.largest = total * perRecord + } + segs = segs[:0] + r.sets++ + } + } + r.segments = len(segs) + sort.Slice(segs, func(i, j int) bool { return segs[i] < segs[j] }) + return r +} diff --git a/database/perm_test.go b/database/perm_test.go new file mode 100644 index 0000000..bf409f4 --- /dev/null +++ b/database/perm_test.go @@ -0,0 +1,192 @@ +package blockchainDB + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func sealPerm(t *testing.T, p *PermStore, height uint64) { + t.Helper() + s, err := p.beginSeal(height) + require.NoError(t, err) + require.NoError(t, s.finish()) +} + +// Records are written once and found in the window; past the window +// Get says absent and GetDeep finds them through the buckets and, +// after a pack, the retired run; the manifest and the deltas replay +// on reopen; a torn delta is dropped whole. +func TestPermStoreTiersAndReopen(t *testing.T) { + every := PermMergeEvery + PermMergeEvery = 4 // 64 buckets a merge + defer func() { PermMergeEvery = every }() + dir := filepath.Join(t.TempDir(), "perm") + p, err := NewPermStore(dir, MinFilterBlocks) + require.NoError(t, err) + fr := NewFastRandom([]byte{21}) + const perBlock = 300 + var keys [][32]byte + value := func(b uint64, i int) []byte { return []byte{byte(b >> 8), byte(b), byte(i >> 8), byte(i), 'v'} } + for b := uint64(1); b <= 3*MinFilterBlocks; b++ { + p.AdvanceBlock(b) + for i := 0; i < perBlock; i++ { + k := fr.NextHash() + keys = append(keys, k) + existing, existed, err := p.PutIfAbsent(k, value(b, i)) + require.NoError(t, err) + require.False(t, existed) + require.Nil(t, existing) + } + // A key written again in its block is refused with its value + existing, existed, err := p.PutIfAbsent(keys[len(keys)-1], []byte("other")) + require.NoError(t, err) + require.True(t, existed) + require.Equal(t, value(b, perBlock-1), existing) + sealPerm(t, p, b) + if b%5 == 0 { + require.NoError(t, p.Merge()) + } + } + // The window holds the last MinFilterBlocks blocks; older keys are + // absent to Get and present to GetDeep + inWindow := keys[2*MinFilterBlocks*perBlock:] + older := keys[:2*MinFilterBlocks*perBlock] + for i, k := range inWindow[:perBlock] { + v, err := p.Get(k) + require.NoError(t, err) + require.Equal(t, value(2*MinFilterBlocks+1, i), v) + } + for _, k := range older[:100] { + _, err := p.Get(k) + require.ErrorIs(t, err, errNotFound, "past the window, absent on the protocol path") + _, err = p.GetDeep(k) + require.NoError(t, err, "and found deep") + } + _, err = p.Get(fr.NextHash()) + require.ErrorIs(t, err, errNotFound) + st := p.Stats() + require.Greater(t, st.HistorySegments, 0, "buckets hold runs") + + // A pack retires the buckets and the pending deltas into one run + require.NoError(t, p.Pack()) + require.Len(t, p.retired, 1) + for b := range p.buckets { + require.Empty(t, p.buckets[b].runs) + } + for _, k := range older[:200] { + _, err = p.GetDeep(k) + require.NoError(t, err, "found in the retired run") + } + + // More blocks, sealed but never in a manifest, then reopen + for b := uint64(3*MinFilterBlocks + 1); b <= 3*MinFilterBlocks+5; b++ { + p.AdvanceBlock(b) + for i := 0; i < perBlock; i++ { + k := fr.NextHash() + keys = append(keys, k) + require.NoError(t, p.Put(k, value(b, i))) + } + sealPerm(t, p, b) + } + // A torn entry after the last seal: the crash + _, err = p.cur.f.WriteAt([]byte{9, 0, 0, 0, 1, 2, 3, 4, 5, 6}, p.cur.size) + require.NoError(t, err) + last := keys[len(keys)-perBlock:] + for _, hf := range p.files { + hf.f.Close() + } + for _, rf := range p.runs { + rf.f.Close() + } + r, err := OpenPermStore(dir) + require.NoError(t, err) + defer r.Close() + require.EqualValues(t, 3*MinFilterBlocks+6, r.height, "the replayed deltas leave the height at the next block, as the seal does") + for i, k := range last { + v, err := r.Get(k) + require.NoError(t, err, "a delta replayed after the manifest") + require.Equal(t, value(3*MinFilterBlocks+5, i), v) + } + for _, k := range older[:100] { + _, err = r.GetDeep(k) + require.NoError(t, err, "the retired run came back through the manifest") + } + _, err = r.Get(fr.NextHash()) + require.ErrorIs(t, err, errNotFound) +} + +// A shard built with both layers as files seals, merges, packs, +// closes and reopens as files, through the same KV2 and KVShard +// surface; permanent keys are found in the window and, once past it, +// by GetDeep; dynamic keys keep their last value. +func TestFilesShardRoundTrip(t *testing.T) { + every := PermMergeEvery + PermMergeEvery = 8 + defer func() { PermMergeEvery = every }() + dir := filepath.Join(t.TempDir(), "shards") + kvs, err := NewKVShardFilesN(dir, 2, 1000) + require.NoError(t, err) + require.NoError(t, kvs.SetFilterBlocks(MinFilterBlocks)) + fr := NewFastRandom([]byte{9}) + hot := make([][32]byte, 100) + for i := range hot { + hot[i] = fr.NextHash() + } + var perm [][32]byte + for b := uint64(1); b <= 3*MinFilterBlocks; b++ { + for _, k := range hot { + require.NoError(t, kvs.PutDyna(k, append([]byte{byte(b)}, k[:4]...))) + } + for i := 0; i < 50; i++ { + k := fr.NextHash() + perm = append(perm, k) + require.NoError(t, kvs.PutPerm(k, append([]byte{byte(b)}, k[:4]...))) + } + require.NoError(t, kvs.SealBlock(b)) + if b%10 == 0 { + require.NoError(t, kvs.Compress()) + _, err := kvs.MergeFinalized(b) + require.NoError(t, err) + } + } + for _, k := range hot { + v, err := kvs.GetDyna(k) + require.NoError(t, err) + require.Equal(t, byte(3*MinFilterBlocks), v[0]) + } + recent := perm[len(perm)-50:] + old := perm[:50] + for _, k := range recent { + _, err := kvs.GetPerm(k) + require.NoError(t, err, "in the window") + } + for _, k := range old { + _, err := kvs.GetPerm(k) + require.ErrorIs(t, err, errNotFound, "past the window") + v, err := kvs.Shards[kvs.ShardIndex(k[:])].GetPermDeep(k) + require.NoError(t, err, "found deep") + require.Equal(t, byte(1), v[0]) + } + _, packed, err := kvs.PackFinalized(3 * MinFilterBlocks) + require.NoError(t, err) + require.True(t, packed) + require.NoError(t, kvs.Close()) + + re, err := OpenKVShard(dir) + require.NoError(t, err) + defer re.Close() + require.NotNil(t, re.Shards[0].Perm, "reopened as files") + require.NotNil(t, re.Shards[0].Heap) + for _, k := range hot { + v, err := re.GetDyna(k) + require.NoError(t, err) + require.Equal(t, byte(3*MinFilterBlocks), v[0]) + } + for _, k := range old { + v, err := re.Shards[re.ShardIndex(k[:])].GetPermDeep(k) + require.NoError(t, err, "the retired run came back") + require.Equal(t, byte(1), v[0]) + } +} diff --git a/database/permindex.go b/database/permindex.go new file mode 100644 index 0000000..5157e09 --- /dev/null +++ b/database/permindex.go @@ -0,0 +1,346 @@ +package blockchainDB + +import ( + "bytes" + "container/heap" + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "io" + "os" + "sort" +) + +// The permanent layer's index primitives (proposal +// docs/proposals/2026-09-16-entries-written-once.md, step 2): a +// record that names where an entry lives, sorted runs of records with +// a filter, a k-way merge over runs, and a lookup in a run. Nothing +// here touches an entry's bytes; entries are written once and only +// ever named. +// +// A run on disk: +// +// header magic "PRUN"(4) count(4) bloomBytes(4) bloomK(4) crc(4) height(8) +// records count sorted 44-byte records: key(32) file(4) off(4) len(4) +// bloom bloomBytes of filter over the keys +// +// Runs are immutable once written; a merge writes a new run from +// several and the inputs are dropped by whoever owns them. + +// permRecord names an entry: its key and where it is. +type permRecord struct { + key [32]byte + file uint32 + off uint32 + n uint32 +} + +const ( + permRecSize = 32 + 4 + 4 + 4 + permRunHdr = 4 + 4 + 4 + 4 + 4 + 8 + permRunMagic = 0x5052554E // "PRUN" + permBloomBits = 12 // Bits per key in a run's filter + permBloomK = 3 +) + +func (r permRecord) put(buf []byte) { + copy(buf, r.key[:]) + binary.LittleEndian.PutUint32(buf[32:], r.file) + binary.LittleEndian.PutUint32(buf[36:], r.off) + binary.LittleEndian.PutUint32(buf[40:], r.n) +} + +func getPermRecord(buf []byte) (r permRecord) { + copy(r.key[:], buf) + r.file = binary.LittleEndian.Uint32(buf[32:]) + r.off = binary.LittleEndian.Uint32(buf[36:]) + r.n = binary.LittleEndian.Uint32(buf[40:]) + return r +} + +// permRun is a sorted run of records with its filter, as held by a +// bucket: the file it lives in, where, and how many; the filter is +// resident when loaded and probed cold otherwise. +type permRun struct { + path string + file uint32 // The run file's id + off int64 // Where the run starts in its file + count uint32 + height uint64 // The block a delta is; the watermark a retired run is; 0 for a bucket's run + bloomAt int64 // Where the filter starts + bloom *Bloom + k int + bytes uint32 + // Every permFenceEvery-th key, for a resident run: a lookup finds + // its chunk in memory and reads that chunk once, where a binary + // search over the file was a pread a probe -- a dozen a run, and + // 40% of a block's CPU under the load platform's reads. + fence [][32]byte +} + +// permFenceEvery is the records between fence keys: a chunk is one +// read of this many records, and the fence costs 32 bytes per chunk. +const permFenceEvery = 32 + +// buildFence takes every permFenceEvery-th key from records laid out +// in buf. +func buildFence(buf []byte, count uint32) [][32]byte { + fence := make([][32]byte, 0, (int(count)+permFenceEvery-1)/permFenceEvery) + for i := 0; i < int(count); i += permFenceEvery { + var k [32]byte + copy(k[:], buf[i*permRecSize:]) + fence = append(fence, k) + } + return fence +} + +// writePermRun writes records, which must be sorted by key and free +// of duplicates, as a run appended to w, and returns it. The caller +// fsyncs the file. +func writePermRun(w io.WriterAt, at int64, file uint32, recs []permRecord, height uint64) (*permRun, error) { + if !sort.SliceIsSorted(recs, func(i, j int) bool { return bytes.Compare(recs[i].key[:], recs[j].key[:]) < 0 }) { + return nil, errors.New("perm run: records are not sorted") + } + bloom := NewBloomSizedForKeys(uint64(len(recs)), permBloomK) + buf := make([]byte, permRunHdr+len(recs)*permRecSize+int(bloom.NumBytes)) + binary.LittleEndian.PutUint32(buf, permRunMagic) + binary.LittleEndian.PutUint32(buf[4:], uint32(len(recs))) + binary.LittleEndian.PutUint32(buf[8:], uint32(bloom.NumBytes)) + binary.LittleEndian.PutUint32(buf[12:], uint32(bloom.K)) + binary.LittleEndian.PutUint64(buf[20:], height) + p := permRunHdr + for i, r := range recs { + if i > 0 && recs[i-1].key == r.key { + return nil, errors.New("perm run: duplicate key") + } + r.put(buf[p:]) + bloom.Set(r.key) + p += permRecSize + } + copy(buf[p:], bloom.Map) + binary.LittleEndian.PutUint32(buf[16:], crc32.ChecksumIEEE(buf[permRunHdr:])) + if _, err := w.WriteAt(buf, at); err != nil { + return nil, err + } + return &permRun{path: permRunName(file), file: file, off: at, count: uint32(len(recs)), height: height, bloomAt: at + int64(p), bloom: bloom, k: bloom.K, bytes: uint32(len(buf)), + fence: buildFence(buf[permRunHdr:], uint32(len(recs)))}, nil +} + +// openPermRun reads a run's header at off in f and verifies the run. +func openPermRun(f *os.File, file uint32, off int64, resident bool) (*permRun, error) { + path := permRunName(file) + hdr := make([]byte, permRunHdr) + if _, err := f.ReadAt(hdr, off); err != nil { + return nil, err + } + if binary.LittleEndian.Uint32(hdr) != permRunMagic { + return nil, fmt.Errorf("perm run at %s:%d: bad magic", path, off) + } + r := &permRun{path: path, file: file, off: off, count: binary.LittleEndian.Uint32(hdr[4:]), k: int(binary.LittleEndian.Uint32(hdr[12:])), height: binary.LittleEndian.Uint64(hdr[20:])} + bloomBytes := binary.LittleEndian.Uint32(hdr[8:]) + r.bloomAt = off + permRunHdr + int64(r.count)*permRecSize + r.bytes = permRunHdr + r.count*permRecSize + bloomBytes + body := make([]byte, r.bytes-permRunHdr) + if _, err := f.ReadAt(body, off+permRunHdr); err != nil { + return nil, err + } + if crc32.ChecksumIEEE(body) != binary.LittleEndian.Uint32(hdr[16:]) { + return nil, fmt.Errorf("perm run at %s:%d: checksum failed", path, off) + } + if resident { + // Rebuilt from the stored byte count: ByteMask indexes by NumBytes + r.bloom = &Bloom{NumBytes: uint64(bloomBytes), SizeOfMap: float64(bloomBytes) / (1 << 20), K: r.k, + Map: make([]byte, bloomBytes), Capacity: uint64(bloomBytes) * 8 / BloomBitsPerKey, Count: uint64(r.count)} + copy(r.bloom.Map, body[int64(r.count)*permRecSize:]) + r.fence = buildFence(body, r.count) + } + return r, nil +} + +// lookup finds key in the run: the filter first (resident, or cold in +// the file), then a binary search over the records. +func (r *permRun) lookup(f *os.File, key [32]byte) (rec permRecord, found bool, err error) { + if r.bloom != nil { + if !r.bloom.Test(key) { + return rec, false, nil + } + } else if ok, err := r.bloomTestCold(f, key); err != nil || !ok { + return rec, false, err + } + if r.fence != nil { + // The chunk whose first key is the greatest not above key + i := sort.Search(len(r.fence), func(i int) bool { return bytes.Compare(r.fence[i][:], key[:]) > 0 }) - 1 + if i < 0 { + return rec, false, nil + } + first := int64(i) * permFenceEvery + n := int64(permFenceEvery) + if first+n > int64(r.count) { + n = int64(r.count) - first + } + buf := make([]byte, n*permRecSize) + if _, err := f.ReadAt(buf, r.off+permRunHdr+first*permRecSize); err != nil { + return rec, false, err + } + for j := int64(0); j < n; j++ { + switch c := bytes.Compare(buf[j*permRecSize:j*permRecSize+32], key[:]); { + case c == 0: + return getPermRecord(buf[j*permRecSize:]), true, nil + case c > 0: + return rec, false, nil + } + } + return rec, false, nil + } + lo, hi := int64(0), int64(r.count) + buf := make([]byte, permRecSize) + for lo < hi { + mid := (lo + hi) / 2 + if _, err := f.ReadAt(buf, r.off+permRunHdr+mid*permRecSize); err != nil { + return rec, false, err + } + switch c := bytes.Compare(buf[:32], key[:]); { + case c == 0: + return getPermRecord(buf), true, nil + case c < 0: + lo = mid + 1 + default: + hi = mid + } + } + return rec, false, nil +} + +// bloomTestCold probes the run's filter in the file: k one-byte +// reads, the way the segment store probes a cold filter. +func (r *permRun) bloomTestCold(f *os.File, key [32]byte) (bool, error) { + bloomBytes := r.bytes - permRunHdr - r.count*permRecSize + probe := &Bloom{NumBytes: uint64(bloomBytes), SizeOfMap: float64(bloomBytes) * 8, K: r.k} + one := make([]byte, 1) + for i := 0; i < r.k; i++ { + idx, mask := probe.ByteMask(key, i) + if _, err := f.ReadAt(one, r.bloomAt+int64(idx)); err != nil { + return false, err + } + if one[0]&mask == 0 { + return false, nil + } + } + return true, nil +} + +// records reads a run's records in order, for a merge. +func (r *permRun) records(f *os.File) ([]permRecord, error) { + buf := make([]byte, int64(r.count)*permRecSize) + if _, err := f.ReadAt(buf, r.off+permRunHdr); err != nil { + return nil, err + } + recs := make([]permRecord, r.count) + for i := range recs { + recs[i] = getPermRecord(buf[i*permRecSize:]) + } + return recs, nil +} + +// recordsBucketRange reads the records whose first key byte lies in +// [lo, hi]: the run is sorted, so they are one slice, found by a +// binary search that probes a byte per step and read at once. A +// merge takes its due buckets' records from every pending delta this +// way rather than reading each delta whole -- with buckets due in +// rotation that was a read of every pending delta, up to +// PermMergeEvery blocks of them, at every pass. +func (r *permRun) recordsBucketRange(f *os.File, lo, hi byte) ([]permRecord, error) { + first := func(i int) (byte, error) { + var b [1]byte + _, err := f.ReadAt(b[:], r.off+permRunHdr+int64(i)*permRecSize) + return b[0], err + } + // The first record whose key's first byte is not below target + search := func(target int) (int, error) { + i, j := 0, int(r.count) + for i < j { + m := int(uint(i+j) >> 1) + b, err := first(m) + if err != nil { + return 0, err + } + if int(b) < target { + i = m + 1 + } else { + j = m + } + } + return i, nil + } + i, err := search(int(lo)) + if err != nil { + return nil, err + } + j, err := search(int(hi) + 1) + if err != nil { + return nil, err + } + if j <= i { + return nil, nil + } + buf := make([]byte, (j-i)*permRecSize) + if _, err := f.ReadAt(buf, r.off+permRunHdr+int64(i)*permRecSize); err != nil { + return nil, err + } + recs := make([]permRecord, j-i) + for k := range recs { + recs[k] = getPermRecord(buf[k*permRecSize:]) + } + return recs, nil +} + +// mergePermRuns merges sorted record lists, oldest first, into one +// sorted list; a key present in several takes the newest. Permanent +// keys are written once, so a duplicate is a replay or a fault, and +// newest-wins matches the store's rule everywhere else. +func mergePermRuns(inputs [][]permRecord) []permRecord { + h := &permMergeHeap{} + total := 0 + for src, in := range inputs { + total += len(in) + if len(in) > 0 { + heap.Push(h, permMergeCursor{src: src, recs: in}) + } + } + out := make([]permRecord, 0, total) + for h.Len() > 0 { + c := heap.Pop(h).(permMergeCursor) + r := c.recs[0] + if n := len(out); n == 0 || out[n-1].key != r.key { + out = append(out, r) + } + // The heap orders equal keys newest first, so the first copy + // out is the one kept and the rest are dropped here + if len(c.recs) > 1 { + heap.Push(h, permMergeCursor{src: c.src, recs: c.recs[1:]}) + } + } + return out +} + +type permMergeCursor struct { + src int + recs []permRecord +} + +type permMergeHeap []permMergeCursor + +func (h permMergeHeap) Len() int { return len(h) } +func (h permMergeHeap) Less(i, j int) bool { + if c := bytes.Compare(h[i].recs[0].key[:], h[j].recs[0].key[:]); c != 0 { + return c < 0 + } + return h[i].src > h[j].src // Newest first on a tie +} +func (h permMergeHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *permMergeHeap) Push(x any) { *h = append(*h, x.(permMergeCursor)) } +func (h *permMergeHeap) Pop() any { old := *h; x := old[len(old)-1]; *h = old[:len(old)-1]; return x } +func sortPermRecords(recs []permRecord) { + sort.Slice(recs, func(i, j int) bool { return bytes.Compare(recs[i].key[:], recs[j].key[:]) < 0 }) +} diff --git a/database/permindex_test.go b/database/permindex_test.go new file mode 100644 index 0000000..056f268 --- /dev/null +++ b/database/permindex_test.go @@ -0,0 +1,95 @@ +package blockchainDB + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// A run round-trips: written sorted with its filter, reopened with the +// checksum verified, looked up resident and cold, merged newest-wins. +func TestPermRunWriteLookupMerge(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, permRunName(7)) + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o644) + require.NoError(t, err) + defer f.Close() + fr := NewFastRandom([]byte{11}) + older := make([]permRecord, 5000) + for i := range older { + older[i] = permRecord{key: fr.NextHash(), file: 1, off: uint32(i * 100), n: 64} + } + sortPermRecords(older) + // The newer run rewrites a tenth of the older keys and adds new ones + newer := make([]permRecord, 0, 1000) + for i := 0; i < 500; i++ { + r := older[i*10] + r.file, r.off = 2, uint32(i) + newer = append(newer, r) + } + for i := 0; i < 500; i++ { + newer = append(newer, permRecord{key: fr.NextHash(), file: 2, off: uint32(1000 + i), n: 32}) + } + sortPermRecords(newer) + + r1, err := writePermRun(f, 0, 7, older, 1) + require.NoError(t, err) + r2, err := writePermRun(f, int64(r1.bytes), 7, newer, 2) + require.NoError(t, err) + _, err = writePermRun(f, int64(r1.bytes+r2.bytes), 7, []permRecord{older[3], older[2]}, 3) + require.Error(t, err, "unsorted records are refused") + + // Reopen: resident and cold + rr, err := openPermRun(f, 7, 0, true) + require.NoError(t, err) + require.EqualValues(t, 5000, rr.count) + require.EqualValues(t, 1, rr.height) + cold, err := openPermRun(f, 7, int64(r1.bytes), false) + require.NoError(t, err) + require.Nil(t, cold.bloom) + for _, rec := range older[:200] { + got, found, err := rr.lookup(f, rec.key) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, rec, got) + } + for _, rec := range newer[:200] { + got, found, err := cold.lookup(f, rec.key) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, rec, got) + } + misses := 0 + for i := 0; i < 2000; i++ { + _, found, err := rr.lookup(f, fr.NextHash()) + require.NoError(t, err) + if !found { + misses++ + } + } + require.Equal(t, 2000, misses, "an absent key is never found") + + // Merge: the newer copy of a rewritten key wins + recs1, err := rr.records(f) + require.NoError(t, err) + recs2, err := cold.records(f) + require.NoError(t, err) + merged := mergePermRuns([][]permRecord{recs1, recs2}) + require.Len(t, merged, 5500) + byKey := map[[32]byte]permRecord{} + for _, r := range merged { + byKey[r.key] = r + } + for _, r := range newer { + require.Equal(t, r, byKey[r.key]) + } + require.EqualValues(t, 1, byKey[older[1].key].file, "an unrewritten key keeps its older record") + + // A damaged run is refused + _, err = f.WriteAt([]byte{0xff, 0xff}, permRunHdr+40) + require.NoError(t, err) + _, err = openPermRun(f, 7, 0, true) + require.ErrorContains(t, err, "checksum") +} diff --git a/database/segstore.go b/database/segstore.go index 25c3175..b842c31 100644 --- a/database/segstore.go +++ b/database/segstore.go @@ -814,6 +814,16 @@ type StoreStats struct { HistorySegments int // Segments in history now ActiveSegments int // Segments in the window now ResidentBloomBytes uint64 // History filter memory held, of BloomResidentBytes + + // The heap's own figures (heap.go), zero for a segment store: the + // key map's resident bytes, its data files, what is live and dead + // in them, and what the mover has scanned and copied so far. + ResidentIndexBytes uint64 + HeapFiles int + HeapLiveBytes uint64 + HeapDeadBytes uint64 + HeapScannedBytes uint64 + HeapMovedBytes uint64 } // storeCounters is StoreStats as the store keeps it: atomics, because @@ -2712,6 +2722,40 @@ func compactionRunWithin(history []*segment, ratio float64, budget uint64) (run // the merge's rule: an uncommitted output sits below the newest active // segment and recoverOrphans deletes it, while the inputs are still // named. +// beginBlockSync and compact are the dynaLayer surface (kv_2.go) +// over Sync and CompactHistory. +func (s *SegmentStore) beginBlockSync() (blockSync, error) { + p, err := s.beginSync() + if err != nil { + return nil, err + } + return p, nil +} + +func (s *SegmentStore) compact(int64) (bool, error) { return s.CompactHistory() } + +// beginPermSeal and mergeBelow are the permLayer surface (kv_2.go) +// over beginSeal and MergeBelow. +func (s *SegmentStore) beginPermSeal(height uint64) (blockSync, error) { + p, err := s.beginSeal(height) + if err != nil { + return nil, err + } + if p == nil { + return nil, nil + } + return segSeal{p}, nil +} + +type segSeal struct{ p *pendingSeal } + +func (s segSeal) finish() error { _, err := s.p.finish(); return err } + +func (s *SegmentStore) mergeBelow(height uint64) (bool, error) { + _, merged, err := s.MergeBelow(height) + return merged, err +} + func (s *SegmentStore) CompactHistory() (compacted bool, err error) { s.maint.Lock() defer s.maint.Unlock() diff --git a/docs/SPEC.md b/docs/SPEC.md index 6a3559d..23decbe 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -94,6 +94,14 @@ Every other rule in this section serves this one. Corollaries: them all put every history probe on the disk — K one-byte reads per segment, on the walk a mutable read takes for any key outside the window (#86, #88). +- A logarithmic cost is acceptable only where its pages can stay + cached. A binary search over an index larger than the page cache + costs a disk read per probe, and the node's own writes evict it over + hours, so the cost follows how long the node has run. The dynamic + layer's lookup is therefore resident: its live keys are the working + set, and a `Get` is one map lookup and one read of the value (the + heap, 1.5; #103). An on-disk binary search belongs only on the + permanent layer's deep reads (1.4). ### 1.3 Blocks, the window, and windowed immutability @@ -527,9 +535,29 @@ resident filter memory; and the count of reads that returned a value other than the last written, which fails the run. A minute whose seal p90 exceeds `-seal-budget` (100 ms) is flagged. +It checks answers, not only times: permanent values derive from their +keys, so every sampled permanent read is checked for presence and +content; checked hot keys must read as last written; and at the end +every store is closed, reopened and every sampled key of both layers +read back. A run that loses one fails (1.8). + What a healthy store must show on it: block time under the interval, the seal bounded and flat with age, put and read p99 flat with age, -and maintenance passes bounded per pass (1.2). A change to the +maintenance passes bounded per pass (1.2), zero wrong answers, and +every store reading back after reopen. + +The instrument has to be sound before a number is the store's. The +disk is shared by every store under test, and an SSD that is never +trimmed stalls every fsync on the box for tens of seconds at a time +under a sustained write load; that reads as a seal tail and is not +one. A run is read with the per-minute heap split and the two-second +store and device timelines (`cmd/bdbench/tools`), which tell a device +stall from a store tail, and a run on a disk that stalls is compared +by its clean minutes or not at all. A disk that never receives +discards will do this, so the measurement disk must be trimmed and +verified trimmed before a run counts; the operator's notes for the +machine a given run was made on are kept with that machine, not in +this repository. A change to the protocol path or to maintenance is measured here before it is measured under Accumulate, because here the seal's wait is the store's own and not the executor's. @@ -542,6 +570,7 @@ The current gaps between Section 1 and the code, in one place: |---|---|---| | #33 | 1.8 one commit point | Residual fsyncs; manifest rewritten whole per commit | | #94 | 1.2, 1.6 bounded maintenance | Compaction passes are bounded by records, not bytes or rate; the seal's barriers wait behind their I/O | +| #103 | 1.2 resident dynamic lookup | The segment-store dynamic layer (`NewKVShard`, what Accumulate opens today) still binary-searches its history runs on disk; the heap (`NewKVShardHeapN`) closes it once the embedder adopts it (accumulate#4546) | A pull request that closes one of these updates this table. A pull request that adds one updates it too — knowingly. diff --git a/docs/proposals/2026-09-16-entries-written-once.md b/docs/proposals/2026-09-16-entries-written-once.md index aadf39f..66e24ac 100644 --- a/docs/proposals/2026-09-16-entries-written-once.md +++ b/docs/proposals/2026-09-16-entries-written-once.md @@ -51,96 +51,255 @@ compaction, merge, pack -- operates on indexes, which are an order of magnitude smaller than what they index (a key and a location, ~40 B, against a ~300 B value) and can be rebuilt from the entries. -## The dynamic layer: a heap with holes +## The dynamic layer: files of entries, a map of keys The dynamic layer holds a bounded key set rewritten forever. Today every rewrite appends and the old copy is garbage until a fold -rewrites everything around it. Instead: - -- **Entries live in a heap file per shard.** An entry is - `[len][key][value][checksum]`; its location is `(file, offset)`. -- **A rewrite that fits reuses the slot.** A value no larger than the - slot it replaces is written in place. The soak's dynamic rewrites - are chain heads, BPT nodes and account state whose size is stable, - so this is the common case and it creates no garbage at all. -- **A rewrite that does not fit appends and frees a hole.** The new - entry goes to the end of the heap (or into a hole that fits); the - old slot becomes a hole. -- **Holes are filled, not swept.** Free space is kept by size class - (`8 << n` bytes); a new entry takes the smallest hole that fits, or - the end of the file. Fragmentation is bounded by the size classes - the way a slab allocator's is, and the store reports the ratio of - hole bytes to live bytes. -- **Moving is the only rewrite, and it is bounded.** When the hole - ratio exceeds a threshold, one pass moves one entry -- the last - live entry of the file into the largest hole that fits -- and stops. - A pass costs one entry, never the file. The file shrinks from the - end when its tail is a hole. +rewrites everything around it, index and filter included. Instead +(`database/heap.go`, opened with `NewKVShardHeapN` / `NewKV2Heap`, +recognised on open by its directory; `bdbench -dyna-heap`): + +- **An entry is written once, at the end of a file.** An entry is + `[len][height][key][value][checksum]` at its exact aligned length; + a key is `(file, offset, length)`. Data files are fixed-size + (`HeapFileBytes`, 16 MB); a block appends to the current file, so + the block sync is one sequential fsync per file the block touched. + (Filling holes at put time was built first and measured: a block's + 11k rewrites scattered over a 380 MB heap dirtied a page each, and + the barrier wrote 4x the ingest -- run 3, 443 MB/s against run 2's + 117. Size classes were measured too: a third of the store wasted + for nothing, once nothing was allocated from free lists.) +- **A rewrite within the block reuses the slot** when the entry keeps + its aligned size: nothing durable names it yet, and the file stays a + contiguous sequence of entries a scan can walk. +- **A rewrite in a later block appends.** The slot the last durable + index names is never overwritten, or a crash between the write and + the block's sync would expose an uncommitted value under a committed + name. The old slot is dead where it lies. +- **The mover makes the space.** On the maintenance cadence a pass + takes the deadest files -- once half dead, or whatever their ratio + while dead bytes exceed live, which bounds the heap at twice its + live set -- copies their live entries into a file of its own, and + marks a file left with nothing live for deletion by the sync after + the delta naming the copies. The mover's file is not the block's, + so their barriers never share an inode (measured: the two fsyncs on + one file flushed each other's pages, seal p50 193 ms). The pass + holds the shard's lock only to pick, to plan and to name, never + across a read, a write or an fsync (measured: reading a 16 MB file + under the lock put the seal at 229 ms). A key rewritten while its + copy is in flight leaves the copy dead on arrival. The store + reports bytes scanned against bytes copied: the heap's write + amplification. - **The key map is in memory** for the live dynamic key set (the - soak's half million keys are ~24 MB per store; 1.2 allows memory - that scales with the working set). It is also what the seal makes - durable, below. + soak's half million keys are ~50 MB per store in Go; 1.2 allows + memory that scales with the working set). What the seal makes + durable is a delta of the keys the block touched; a snapshot starts + a new index generation in a file of its own on the maintenance + cadence, written aside, fsynced and renamed into place, so no delta + is ever truncated away. The layer's size converges to O(live keys) by construction (1.5), -without the deeper fold 2.7 allows today. +without the deeper fold 2.7 allows today, and nothing but entries is +ever copied. ## The permanent layer: append-only data, merged indexes The permanent layer is write-once; it has no garbage. Its merges and -packs exist to bound the file count and the length of a history walk -(2.7). Both are properties of the *index*, so: - -- **A shard's data is one growing file** (or one per `PackEvery` - blocks). A block's records are appended; nothing is ever copied. -- **Each block seals an index delta**: the sorted `(key, offset)` pairs - of the records the block appended, with its filter (2.5). -- **Merge and pack fold index deltas** into one sorted index per shard - per level, on the adapter's cadence, off the protocol path, exactly - as today's tiers do -- but moving ~40 B per record instead of ~300. - The walk length and the file count are bounded by the index tiers; - the data file is never in the walk (a key resolves to an offset, - then one `pread`). +packs exist to bound the file count and the length of a lookup's walk +(2.7), and both are properties of the *index*. Today (`segstore.go`, +`seal.go`, `blockset.go`) each block seals a segment -- a body file and +an index file with an embedded filter -- and `MergeBelow` folds +segments with `concatSegments`, which copies every body byte-verbatim +into a new body and rebuilds the index over it; `PackFinalized` copies +the bodies again into a cross-shard set file. An index record is 48 +bytes, `key, offset, length`, with no file identifier: several bodies +are only ever addressed through an out-of-band base, which is why the +bodies have to be concatenated to be merged. Instead: + +- **A shard's data is a sequence of fixed-size files, appended and + never moved**, the heap's file model (`HeapFileBytes`): a block's + records go to the end of the current file, and a record is + `(file, offset, length)`. Nothing is ever copied; a file is deleted + only by `DropBelow` when a pack no longer needs it -- and packs no + longer copy either, so a data file lives until the retention policy + above the store says otherwise. +- **An index record carries the file.** `key(32) file(4) offset(4) + length(4)`, 44 bytes, sorted by key as today. `mergeIndexes` and + `indexWriter` already merge indexes without touching a body; with + the file in the record they need no bases. +- **Each block seals an index delta** with its filter: the sorted + records of what the block appended, ~40 bytes a record. The delta + is what the window is made of; the live key filters (`keyfilter.go`) + are built over deltas exactly as they are built over segments now. +- **The long search is bucketed, and merged a bucket at a time.** A + shard's index below the window is `PermBuckets` buckets (256 by + default; nothing about the number is special, and a modulus serves + as well as a mask), each holding a few sorted runs of 44-byte + records with a filter per run. Every block the maintenance step + takes the next bucket in rotation, gathers that bucket's records + from the deltas sealed since the bucket was last merged, writes them + as a new run, and folds the bucket's runs by ratio; a delta older + than every bucket's last merge is dropped. So the work every block + is a fixed slice, proportional to what arrived for one bucket, and + the big fold is rare and bounded per pass -- never a rewrite of the + shard's whole index, which would grow with the chain (1.2). A + merge swaps one bucket's runs under that bucket's lock; the other + buckets keep answering. A pack is the same fold across shards for + a block range, with no bodies in it. +- **Reads keep their protocol-path rule** (1.3): the window is the + last N blocks' deltas behind the live filters; an immutable key the + filters deny is absent. Below the window a key's bucket is probed + newest run first, one filter and one binary search per run, and + that is `GetDeep`'s walk; a lookup resolves a key to `(file, + offset, length)` and does one `pread` of the data file. + +### Sizing the buckets + +`TestPermSizingSim` (`go test ./database/ -run TestPermSizingSim -v +-args -sim`) models one shard at the soak's rate, 1,040 permanent +records a block, against the store as it is (segments folded by +copying bodies every 20 blocks under the ratio rule, packed every +1,000 blocks). Its first result was a warning: a bucket that is +never drained holds 1/B of the whole chain, so with any fixed count +the largest fold grows with the age of the store (256 buckets: 14 MB +at a day, 102 at a week, 460 at a month), against 1.2. + +The bound is the pack watermark, which the store already has. Every +`PackEvery` (1,000) blocks the finished history is packed and dropped +from the shard; the buckets cover only the history above the +watermark, and since the keys are hashes their shares are even. A +bucket then holds at most 1,000 blocks of its share -- ~4,000 +records, ~180 KB, at 256 buckets -- so a fold is bounded by the pack +period, not the chain, and nothing needs to split. The count is a +free choice; 256 keeps runs small and a merge's lock small. + +Per shard per block that comes to about: the block's delta (46 KB), +the bucket runs (46 KB), their folds (~46 KB) and the pack amortized +(46 MB of index every 1,000 blocks, 46 KB): ~180 KB a block against +today's 2.5 MB, every byte an index byte. The pack is index-only, +46 MB a shard in place of 358 MB of bodies, and the chain's growth +lives where it lives today: in the sets below the watermark, grouped +by block range with one filter per finished group, reached only by +`GetDeep`. Filters for the buckets and the newest sets stay +resident within a budget; older groups are probed cold, as now. + +Ratio 0.25 stays (0.1 halves the runs a lookup probes but nearly +doubles the bytes written); every bucket is merged every 256 blocks +(B/256 buckets a block), which keeps the recent sections at a few MB +a shard. + +### Shards partition keys, not files (measured) + +The dynamic layer alone, nine stores, five minutes: seal p50 54-65 ms +at 8 storage shards, 67-79 ms at 64, 75-86 ms at 128, with a burst of +lost blocks every other minute at 64 and 128 and 8x and 16x the +files. Every shard's files sync on their own at every block, so the +barrier count rises with the shard count; a hundred-plus shards are +right for spreading the retiring work and shrinking every lock, and +they must not each bring a barrier. A block's records for every +shard go into one per-store append file, shard-tagged, and each +shard's index points into it: one data fsync per store however many +shards. ## The seal: one commit point per store per block -With indexes as the sealed object a block is one barrier round per -store rather than four per shard: +Today a non-empty block costs each shard four barriers -- the data +fsync, the index file's, the manifest's temp file, the directory -- +and the store two more for its block record (`seal.go`, +`commitJSON`). Nine stores of eight shards ask the device for about +360 barriers a second before any maintenance runs, and the platform +measured every seal goroutine waiting in `fsync` (#94). -1. Every shard's data appends (permanent) and slot writes (dynamic) - for the block are fsynced, in parallel: one round. -2. The block's index deltas -- permanent `(key, offset)` and dynamic - `(key, location)` for every key the block touched -- are written to - one per-store block file and fsynced: one round. -3. The store's manifest names the block file: the existing commit, - one round. +With deltas as the sealed object, a block's permanent index delta is +appended to the same data file right after the block's records, and +the dynamic layer's delta after its entries the same way, so that: -Three rounds per store per block, none per shard beyond the parallel -data sync, in place of four rounds per shard. This is 1.8's "one -commit point" and closes #33. +1. Every shard fsyncs its data files once: entries and delta together, + in parallel across shards -- one round. +2. The store's block record names, for every shard, the file and + offset of the block's deltas: the existing `block.json` commit, + temp file and directory -- one round. + +Two rounds per store per block, in place of four per shard plus two. +Recovery trusts a delta only if its own checksum holds and every entry +it names checks, since one fsync does not order the delta's bytes +after the entries'; a delta that fails is the block that did not +commit, exactly as a torn delta is today. This is 1.8's "one commit +point" and closes #33. ## Durability and crash consistency (1.8) -- **A hole is reusable one seal late.** A slot freed in block N may - be reused only after N's seal is durable. Until then the durable - index still names the old slot, and a crash must find it intact. - Reuse is deferred the way 2.6 defers deletion. The adapter never - asks the store for an old version (its pre-images are memoized on - its side), so reuse waits on the seal and on nothing else. +- **A file is deleted one seal late, by the mover, and never while a + delta names it.** A file emptied by the mover leaves the map only + after the delta naming the mover's copies out of it is durable. + Until then the durable index still names its slots, and a crash + must find them intact: never unlink what a durable index names. + The deltas name ranges, and the mover's files are the destination + of theirs: a file every copy in which has died is still named by + the generation's deltas, and stays until the snapshot that + supersedes them, however dead it is. (Measured the other way + first: the platform's reopen check found a store closed cleanly + that would not open, its replay stopped at a delta naming an + unlinked mover file, and the derivation deleting what it then took + for unnamed.) Each file carries the bytes such deltas name in it; + the pinned-bytes rule counts those files, so the snapshot comes + early when they pile up. The unlink itself is the mover's, at + its next pass, so no block's seal holds the store lock over a + directory operation; between the two the file is unnamed on disk, + which an open deletes as such. The adapter never asks the store + for an old version (its pre-images are memoized on its side), so + deletion waits on the seal and on nothing else. +- **A seal is one fsync per file it touched, outside the lock.** The + block's delta is written into the data file behind the block's + entries before the fsync, so one barrier covers both, in both + layers; replay trusts the last delta only if every entry it names + checks. A put that lands during the seal belongs to the next + block: the seal takes the block's records under the lock and + releases it before the barrier. +- **What the manifest does not name is still replayed if a seal + wrote it.** The permanent layer's manifest is committed by + maintenance, not by seals, so the seals roll data files the + manifest has not seen. Open takes data files in id order past the + manifest's next id for as long as they exist, and replays their + deltas; a run file past the manifest's next run id is maintenance + output a crash left unnamed, which open removes, so the id is free + for exclusive creation again and no file is ever published over + (1.7). A merge lost this way is done again from the pending + deltas, which is why the manifest need only be committed every + eighth merge, or when a fold leaves run files to drop. - **A torn slot is detected, not misread.** Every entry carries its - length and a checksum; a slot whose checksum fails after a crash is - a hole, and the durable index never points at one, because an index - entry is durable only after the slot it names is. -- **The in-memory key map is rebuilt on open** from the last durable - index snapshot plus the block files after it, the same replay 2.8 - does for manifests today. A snapshot is taken on the pack cadence - so the replay is bounded. -- **Nothing durable is ever overwritten by a different entry** except - a slot the durable index no longer names. 1.7's identity rule - holds for files: a data file is appended, never republished. + length and a checksum; an entry above the committed height is a + block that never synced. An index entry is durable only after the + slot it names is, so a torn slot is never named; open cuts the + files back to their last named slot and drops a torn delta whole. +- **The key map is rebuilt on open** from the newest whole index + generation: its snapshot, then its deltas. A generation whose + write was interrupted is not whole and is removed. +- **Repair reads the keys from the data.** With the index gone, a + scan of the data files rebuilds the map: every entry carries its + key and the height that wrote it, so the highest committed copy of + a key wins (later in the scan on a tie) and an entry above the + committed height, which only the store above knows, is dropped. A + child process killed mid-block, three times over, reopens and + repairs to exactly the last durable block (`heap_crash_test.go`). +- **Nothing durable is ever overwritten by a different entry.** 1.7's + identity rule holds for files: a data file is appended, never + republished. ## What it is measured with -`cmd/bdbench -stores 9`, the run above, is the acceptance test: +`cmd/bdbench -stores 9`, the run above, is the acceptance test. It +checks answers, not only times: a permanent value is derived from its +key, so every sampled permanent read (through the deep read, as the +adapter reads anything older than the window) is checked for presence +and content, a checked hot key must read as last written, and at the +end every store is closed, reopened and every sampled key of both +layers read back -- a run that lost one fails. Until 2026-09-17 the +platform tolerated "not found" on every permanent read, and a data +file rolled between manifest commits was being lost on reopen without +a number moving. With the check in place: 200,000 sampled permanent +keys per store read back correctly after reopen, zero mismatches +under load. The acceptance run must show: - seal p90 under `-seal-budget` (100 ms) and flat from minute 1 to minute 30; @@ -152,10 +311,139 @@ commit point" and closes #33. ## Order of work -1. The dynamic heap, behind the existing `KV2` dynamic surface (`Put`, - `Get`, `Seal`, `CompactHistory` becoming the bounded move, - `Stats`), so the sharding and the adapter do not change. The - platform measures it alone (`-stores 9 -perm 0`). -2. The permanent index deltas and the single block file, which also - brings the seal to one commit point. -3. Merge and pack over indexes. +1. The dynamic heap, behind the existing `KV2` dynamic surface, so + the sharding and the adapter do not change. *Built (PR #97).* + Alone on the disk with nine stores (run 7): seal p50 53-61 ms in + every minute, read p99 1-2 µs, maintenance ~10 s a minute, store + 3.8 GB at five minutes, zero wrong answers; the segment layer alone + seals at 33-38 ms p50 but reads at 11-13 µs, spends 22-110 s a + minute compacting, and had a compaction storm in minute 4 (seal + max 1.5 s, 58 blocks missed). The heap's remaining tail (p90 + 150-250 ms in the minutes the mover copies most) is the mover's + own fsync volume in the device queue, which the pass size paces. +2. The permanent layer as files of records with index deltas + (`PermStore`, behind `KV2`'s permanent surface), the 44-byte index + record, the bucketed long search merged in rotation, and packs over + indexes. *First cut built and wired (`perm.go`, + `NewKVShardFilesN`, `bdbench -perm-files`); its first full-load run + put the seal at 59 ms p50 in minute 1, then climbing to 209 by + minute 5 because maintenance shared the seal's run file; fixed, + remeasured next.* Measured + alone first (`-stores 9 -dyna 0`), then with the heap under the + full load, which is the acceptance run. +3. The block's deltas in the data files: one barrier per shard per + layer. *Built for both layers.* The heap's delta names no key: + a block's entries are contiguous in its file and carry their + keys, so the delta is the ranges the block wrote (the mover's + named copies first, then the block's own) and the copies that + arrived dead, a few dozen bytes a block; a delta of 44-byte + records was a fifth of the heap's writes. Replay scans the + ranges and trusts the last delta only if every entry in it + checks. The permanent layer's delta stays a run with its filter, + since it is the window's search structure, but lives in the data + file too. Alone, the heap seals at 44-54 ms p50 at nine stores + (57-64 with two barriers). The mover finds a file's live entries + through the index rather than by scanning the file, because nine + stores' scans together starved the block loops for CPU. The + mover's budget is the store's (`HeapStoreCleanBytes`), shared + among its shards: with one shard and a shard's budget the store + grew to 6 GB in five minutes. A merge reads only its due + buckets' slice of each pending delta (the run is sorted, the + slice is found by a binary search on the first key byte) instead + of every pending delta whole at every pass. + + *Measured, 2026-09-17:* every "bad minute" of the day's runs -- + seal p50 unchanged, p90 200-450 ms, the heap's fsync average + 3-8x its usual 12-22 ms -- was the disk, not the store. A + 2-second timeline showed every fsync on the box stepping to + 100-330 ms for 30-40 s at once, with nothing in the store + changing (no snapshot, the same mover volume and release cadence) + and with one shard per store (9 barriers/s) exactly as with eight + (72/s). The instrument's root volume is on LUKS without discards + and has never been trimmed. Until it is, runs are compared by + their clean minutes; the per-minute row carries the heap's split + (fsync average and bytes, snapshots, releases, moved bytes) and + each store's seal p90 so a stall can be told from a tail. + + *Maintenance is a slice per call, in rotation.* The heap's + remaining tail in clean minutes was the mover's own delivery: on + the adapter's cadence every shard of every store ran its pass in + the same second -- 300 MB and 72 barriers at once at nine stores + -- and the seals' fsyncs queued behind it for the next four + seconds (device write time 24 s in a 2-second window against 1-3 + otherwise). A store's mover rate is now `HeapStoreCleanBytes` per + `HeapCleanPeriod` blocks, and a `Compress` call takes the shards + next in rotation, as many as the blocks since the last call earn, + each with its share: called every block it moves a little on one + shard, and a hundred shards spread the same rate a hundred ways + with one shard locked at a time. The permanent layer's merge + already worked this way (buckets due by blocks elapsed). + Snapshots are by block count, or early when the files held only + by their deltas outweigh two data files; the manifest commits by + block count. + + *Measured on that build (before the every-block cadence), nine + stores, eight shards, clean minutes:* the heap alone 36-38 ms p50 + / 48-60 p90 in every minute; the files store 36-38 / 50-52, where + the day before it was 60 / 370-450 in every minute after the first + -- the seal's second barrier, the merge's three, and its reread of + every pending delta were the tail. One shard per store: 37-39 / + 51-54. + + *What waits unmerged is bounded by the window.* A bucket was + merged once per 256 blocks, so up to 256 deltas waited unmerged, + and every deep read probed each one's filter and every merge read + a slice of each: over five minutes read p99 climbed from 7 to 47 + us and merge work from 157 to 305 s a minute. A bucket is now + merged once a window (`PermMergeEvery` = `MinFilterBlocks`), so + no more than a window of deltas waits; the extra folds are a + fraction of the index bytes, themselves a sixth of the data. + + *The cadence per layer.* On the every-block cadence the heap's + slice is cheap, but the permanent layer's merge ended every call + with an fsync of its run file: at a call per shard per block that + was 72-144 barriers a second at nine stores, merges cost 190-250 s + of work a minute, and every seal's fsync doubled. A bucket's run + is named by the manifest alone, so it needs durability before the + manifest commit and not before: the run files are fsynced at the + commit (every PermManifestBlocks, or at a fold), and a merge call + has no barrier. + + *Measured with every fix above, both layers, nine stores, eight + shards, maintenance every block, 2026-09-17 00:05:* seal 38-42 ms + p50 and 57-65 p90 in the four clean minutes (the fifth was a disk + stall), merges 3-8 s of work a minute, zero mismatches, and every + store reopened with all 200,000 sampled permanent keys and 1,024 + checked dynamic keys read back correctly. Block time drifted from + 157 to 210 ms: the profile put 40% of the CPU in the deep read's + binary search over bucket runs, a pread a probe; a resident run + now keeps a fence of every 32nd key and a lookup is one read. + Measured: read p99 6-7 us instead of 9-10, block time flat at + 178-181 ms from minute 3 on instead of climbing, all nine stores + read back clean again. What remains in the deep read is one read + of the run's chunk and one of the entry, which is the floor for + an index that does not fit in memory. + + Still to do: the store-level commit (one block record naming + every shard's deltas) and the per-store data file, so that a + hundred shards cost a block one barrier; one shard per store is + that layout by another name and measures the same as eight now. + The 30-minute acceptance run (2026-09-17, after the disk was + trimmed; nine files stores, maintenance every block, phase + offset): 16,164 blocks, every minute's seal p90 between 42 and + 57 ms against the 100 ms budget, the heap's fsync average flat + at 10-14 ms, no mismatch, and all nine stores reopened and read + every sampled key back with 0 wrong. The device's write ticks + per two-second window were p50 1.9 s, max 14.3 s: no stall. + Block p50 rose from 129 ms to ~170 ms over the first five + minutes as the window filled and then held at 170-193 ms. In + minutes 28 and 30 the process read 56 and 120 MB/s from the + drive and read p99 went from 7-8 us to 52-59 us, which at 40,000 + lookups a block put block p50 at 250-300 ms and p90 at 370-460 + ms; the seal did not move. The store was 68-74 GB on a 64 GB + box whose page cache had been full since minute 10: lookups of + permanent keys of all ages had begun to miss the cache. That + is the floor for an index that does not fit in memory, not a + cost that grows with the store's age in the store's own work, + and it is where a resident permanent index (or a bigger box) + would show. Run: accept-30m, 2026-09-17 08:00.