Move design docs and specs next to the code that cites them

- Move the long-turn memory benchmark guide to
  `benchmarks/long-turn-memory.md`, next to its script, so the repository no
  longer has a top-level `docs/` folder.
- Add the session manager design record and its TLA+ models and trace
  checkers under `src/core/session_manager/`, where the session code already
  cites them.
- Add the indexer refresh model under `src/core/indexer/tla/`.
- Point spec citations in code comments at the new paths, and remove
  references to design notes that are not in the repository.
This commit is contained in:
Pranit committed 2026-10-13 20:46:43 -04:00
1 parent 7e8b85c359
commit eae560291b
82 files changed
+6228 -25

No files matched your search

File renamed without changes.
+1 -1
View File
@@ -877,7 +877,7 @@ fn resolveSubagentAuthority(
}
/// Joins the child threads before the session they append to can close
/// (`tla/Wiring.tla` ParentOutlivesChildren).
/// (`src/core/session_manager/tla/Wiring.tla` ParentOutlivesChildren).
pub fn disableSubagentHost(state: *ServerState) void {
if (state.subagent_host) |host| {
host.deinit();
+1 -1
View File
@@ -3824,7 +3824,7 @@ pub fn Runtime(comptime App: type) type {
}
/// Stops the subagent host, joining its child threads, before the
/// session they append to can close (`tla/Wiring.tla`
/// session they append to can close (`src/core/session_manager/tla/Wiring.tla`
/// ParentOutlivesChildren).
pub fn disableSubagentHost(app: *App) void {
if (app.session_persistence.subagent_host) |host| {
+1 -1
View File
@@ -9,7 +9,7 @@ const io_mod = @import("../shared/io.zig");
const Allocator = std.mem.Allocator;
/// D7: the index is never read beyond this size.
/// The index is never read beyond this size.
const max_index_bytes: usize = 128 * 1024 * 1024;
const mode_type_mask: u32 = 0o170000;
+1 -1
View File
@@ -8,7 +8,7 @@ const wildmatch = @import("wildmatch.zig");
const Allocator = std.mem.Allocator;
/// Largest pattern file read, per source (D7).
/// Largest pattern file read, per source.
pub const max_pattern_file_bytes: usize = 1024 * 1024;
const Pattern = struct {
+3 -3
View File
@@ -29,12 +29,12 @@ pub const Kind = tree_mod.Kind;
pub const Entry = tree_mod.Entry;
pub const Options = struct {
/// D7: at most this many files are returned. Folder rows do not count;
/// At most this many files are returned. Folder rows do not count;
/// the collection limit (`collect_factor`) bounds them.
candidate_cap: usize = 100_000,
max_path_bytes: usize = 2048,
/// Folder and file names skipped outside repositories and below an
/// ignored root (D4, U3). Never applied inside a repository.
/// ignored root. Never applied inside a repository.
skipped_names: []const []const u8 = &default_skipped_names,
max_workers: usize = 8,
};
@@ -634,7 +634,7 @@ fn entryLessThan(_: void, a: Entry, b: Entry) bool {
}
/// Length of the sorted prefix that holds the first `cap` files, or null when
/// `entries` has no more than `cap` files. Folder rows do not count (D7).
/// `entries` has no more than `cap` files. Folder rows do not count.
fn capCut(entries: []const Entry, cap: usize) ?usize {
var files: usize = 0;
for (entries, 0..) |entry, index| {
+74
View File
@@ -0,0 +1,74 @@
-------------------------------- MODULE Refresh --------------------------------
EXTENDS Naturals, Sequences
CONSTANTS MaxSteps, Guarded
Files == {"a.txt", "b.txt"}
VARIABLES fs, tick, stamp, ignoreB, ignoreEpoch, knownStamp, knownIgnore,
reusable, snapshot, phase, beforeStamp, beforeIgnore, beforeTick,
listed, published, history
vars == <<fs, tick, stamp, ignoreB, ignoreEpoch, knownStamp, knownIgnore,
reusable, snapshot, phase, beforeStamp, beforeIgnore, beforeTick,
listed, published, history>>
Visible == IF ignoreB THEN fs \ {"b.txt"} ELSE fs
Init == /\ fs = {} /\ tick = 0 /\ stamp = 0
/\ ignoreB = FALSE /\ ignoreEpoch = 0
/\ knownStamp = 0 /\ knownIgnore = 0 /\ reusable = FALSE
/\ snapshot = {} /\ phase = "idle" /\ beforeStamp = 0
/\ beforeIgnore = 0 /\ beforeTick = 0 /\ listed = {}
/\ published = FALSE /\ history = <<>>
Edit(p) == /\ fs' = IF p \in fs THEN fs \ {p} ELSE fs \cup {p}
/\ stamp' = tick /\ published' = FALSE
/\ history' = Append(history, "Edit:" \o p)
/\ UNCHANGED <<tick, ignoreB, ignoreEpoch, knownStamp, knownIgnore,
reusable, snapshot, phase, beforeStamp,
beforeIgnore, beforeTick, listed>>
Tick == /\ tick < 2 /\ tick' = tick + 1 /\ published' = FALSE
/\ history' = Append(history, "Tick")
/\ UNCHANGED <<fs, stamp, ignoreB, ignoreEpoch, knownStamp, knownIgnore,
reusable, snapshot, phase, beforeStamp, beforeIgnore,
beforeTick, listed>>
ChangeIgnore == /\ ignoreB' = ~ignoreB /\ ignoreEpoch' = ignoreEpoch + 1
/\ published' = FALSE
/\ history' = Append(history, "ChangeIgnore")
/\ UNCHANGED <<fs, tick, stamp, knownStamp, knownIgnore,
reusable, snapshot, phase, beforeStamp,
beforeIgnore, beforeTick, listed>>
Reuse == /\ phase = "idle" /\ stamp = knownStamp
/\ (~Guarded \/ (reusable /\ stamp < tick /\
ignoreEpoch = knownIgnore))
/\ published' = TRUE /\ history' = Append(history, "Reuse")
/\ UNCHANGED <<fs, tick, stamp, ignoreB, ignoreEpoch, knownStamp,
knownIgnore, reusable, snapshot, phase, beforeStamp,
beforeIgnore, beforeTick, listed>>
Begin == /\ phase = "idle" /\ phase' = "listing"
/\ beforeStamp' = stamp /\ beforeIgnore' = ignoreEpoch
/\ beforeTick' = tick /\ published' = FALSE
/\ history' = Append(history, "Begin")
/\ UNCHANGED <<fs, tick, stamp, ignoreB, ignoreEpoch, knownStamp,
knownIgnore, reusable, snapshot, listed>>
List == /\ phase = "listing" /\ phase' = "validate" /\ listed' = Visible
/\ published' = FALSE /\ history' = Append(history, "List")
/\ UNCHANGED <<fs, tick, stamp, ignoreB, ignoreEpoch, knownStamp,
knownIgnore, reusable, snapshot, beforeStamp,
beforeIgnore, beforeTick>>
Commit == LET valid == ~Guarded \/ (stamp = beforeStamp /\
ignoreEpoch = beforeIgnore /\
stamp < beforeTick)
IN /\ phase = "validate" /\ phase' = "idle"
/\ snapshot' = IF valid THEN listed ELSE snapshot
/\ knownStamp' = IF valid THEN stamp ELSE knownStamp
/\ knownIgnore' = IF valid THEN ignoreEpoch ELSE knownIgnore
/\ reusable' = IF valid THEN TRUE ELSE reusable
/\ published' = valid /\ history' = Append(history, "Commit")
/\ UNCHANGED <<fs, tick, stamp, ignoreB, ignoreEpoch, beforeStamp,
beforeIgnore, beforeTick, listed>>
Next == /\ Len(history) < MaxSteps
/\ ((\E p \in Files : Edit(p)) \/ Tick \/ ChangeIgnore \/
Reuse \/ Begin \/ List \/ Commit)
Spec == Init /\ [][Next]_vars
TypeOK == /\ fs \subseteq Files /\ snapshot \subseteq Files
/\ listed \subseteq Files /\ tick \in 0..2 /\ stamp \in 0..2
/\ ignoreB \in BOOLEAN /\ published \in BOOLEAN
/\ reusable \in BOOLEAN /\ phase \in {"idle", "listing", "validate"}
/\ ignoreEpoch \in 0..MaxSteps /\ Len(history) <= MaxSteps
NoStalePublication == published => snapshot = Visible
=============================================================================
+21
View File
@@ -0,0 +1,21 @@
---------------------------- MODULE RefreshWitness ----------------------------
\* Each invariant claims a reuse path cannot happen. TLC's counterexample is
\* then the shortest trace that exercises it, which the real replay runs.
EXTENDS Refresh
Before(a, b) == \E i, j \in 1..Len(history) : i < j /\ history[i] \in a /\ history[j] = b
NoReuse == ~Before({"Begin"}, "Reuse")
NoReuseAfterEdit == ~Before({"Edit:a.txt", "Edit:b.txt"}, "Reuse")
\* Real files have only times, so an ignore change counts once a Tick passes;
\* b.txt exists first so the reused snapshot must reflect the new rule.
NoReuseAfterIgnore == ~\E i, k, m, j \in 1..Len(history) :
/\ i < k /\ k < m /\ m < j
/\ history[i] = "Edit:b.txt" /\ history[k] = "ChangeIgnore"
/\ history[m] = "Tick" /\ history[j] = "Reuse"
\* A change after a reuse: the real code must notice it at the next check.
NoEditAfterReuse == ~\E i, j \in 1..Len(history) :
i < j /\ history[i] = "Reuse" /\ history[j] \in {"Edit:a.txt", "Edit:b.txt"}
NoIgnoreChangeAfterReuse == ~\E i, k, j \in 1..Len(history) :
/\ i < k /\ k < j
/\ history[i] = "Edit:b.txt" /\ history[k] = "Reuse"
/\ history[j] = "ChangeIgnore"
=============================================================================
@@ -0,0 +1,4 @@
SPECIFICATION Spec
CONSTANTS MaxSteps = 6
Guarded = FALSE
INVARIANTS TypeOK NoStalePublication
@@ -0,0 +1,3 @@
SPECIFICATION Spec
CONSTANTS MaxSteps = 10 Guarded = TRUE
INVARIANTS TypeOK NoStalePublication
@@ -0,0 +1,3 @@
SPECIFICATION Spec
CONSTANTS MaxSteps = 10 Guarded = TRUE
INVARIANTS TypeOK NoStalePublication NoEditAfterReuse
@@ -0,0 +1,3 @@
SPECIFICATION Spec
CONSTANTS MaxSteps = 10 Guarded = TRUE
INVARIANTS TypeOK NoStalePublication NoIgnoreChangeAfterReuse
@@ -0,0 +1,3 @@
SPECIFICATION Spec
CONSTANTS MaxSteps = 10 Guarded = TRUE
INVARIANTS TypeOK NoStalePublication NoReuse
@@ -0,0 +1,3 @@
SPECIFICATION Spec
CONSTANTS MaxSteps = 10 Guarded = TRUE
INVARIANTS TypeOK NoStalePublication NoReuseAfterEdit
@@ -0,0 +1,3 @@
SPECIFICATION Spec
CONSTANTS MaxSteps = 10 Guarded = TRUE
INVARIANTS TypeOK NoStalePublication NoReuseAfterIgnore
+4
View File
@@ -0,0 +1,4 @@
SPECIFICATION Spec
CONSTANTS MaxSteps = 6
Guarded = TRUE
INVARIANTS TypeOK NoStalePublication
+1 -1
View File
@@ -4,7 +4,7 @@
//! its times are older than the scan's start by the granularity margin, so a
//! change in the same clock tick as the scan, which a later stamp comparison
//! could not see, always forces a new scan. This is the reuse certificate the
//! refresh model (Refresh.tla) requires.
//! refresh model (`tla/Refresh.tla`) requires.
const std = @import("std");
const io_mod = @import("../shared/io.zig");
+1 -1
View File
@@ -1160,7 +1160,7 @@ pub const Runtime = struct {
}
/// Permission is fx's tool admission, before the call reaches the
/// runtime, and only the runtime calls the engine (D6).
/// runtime, and only the runtime calls the engine.
fn allow(_: *anyopaque, _: []const u8, _: []const u8, _: ?[]const u8) bool {
return true;
}
+9 -8
View File
@@ -18,7 +18,7 @@
//! ACP client's prompt and tool identities are settings (D46); hosted
//! terminal state lives in `~/.fx/terminal/{id}/` (D45). An older session
//! with a `~/.fx/session-files/{id}/` side folder moves on its first
//! writable open (D47, `tla/MoveSideFiles.tla`).
//! writable open (D47, `src/core/session_manager/tla/MoveSideFiles.tla`).
const std = @import("std");
const testing_allocator = @import("../shared/testing_allocator.zig");
@@ -1123,7 +1123,7 @@ pub const Session = struct {
}
/// Closes the session and frees the adapter. Every child thread of this
/// session must have joined (`tla/Wiring.tla` ParentOutlivesChildren).
/// session must have joined (`src/core/session_manager/tla/Wiring.tla` ParentOutlivesChildren).
/// A copy of its capability that a background reader still holds keeps
/// reading, and stores nothing more (D44).
pub fn close(self: *Session) void {
@@ -1782,11 +1782,12 @@ pub const Session = struct {
// -- the move off the side folder (D47) -----------------------------------
/// Makes an older session one manager folder (D47, `tla/MoveSideFiles.tla`)
/// on its first writable open, then loads the map its old handles resolve
/// through. Every body in the side folder becomes a blob, the map from
/// each old name to its blob is one more blob, and one batch records the
/// map, every moved blob, and the client's settings from the old files.
/// Makes an older session one manager folder (D47,
/// `src/core/session_manager/tla/MoveSideFiles.tla`) on its first writable
/// open, then loads the map its old handles resolve through. Every body
/// in the side folder becomes a blob, the map from each old name to its
/// blob is one more blob, and one batch records the map, every moved
/// blob, and the client's settings from the old files.
/// Then the terminal folder moves to `~/.fx/terminal/{id}` and the side
/// folder goes. Each step can be repeated, so a crash at any point
/// redoes the move on the next open.
@@ -1813,7 +1814,7 @@ pub const Session = struct {
}
/// Every step of the move but the side folder's removal, in the order
/// `tla/MoveSideFiles.tla` checks; returns how many bodies moved.
/// `src/core/session_manager/tla/MoveSideFiles.tla` checks; returns how many bodies moved.
fn moveOut(self: *Session, side: *io_mod.VerifiedDir) !usize {
var scratch = std.heap.ArenaAllocator.init(self.alloc);
defer scratch.deinit();
+1 -1
View File
@@ -3,7 +3,7 @@
//!
//! This file (L4) is the only public root; fx imports it and nothing else.
//! A process creates one `Manager` with `init`, which does no disk I/O.
//! Thirteen operations cover every host (docs/session-manager/system-design.md
//! Thirteen operations cover every host (`system-design.md`
//! "API"), plus `verify` and `rebuild` for doctor:
//!
//! openNew openResume openFork openImport -> Session
File diff suppressed because it is too large. Load diff
+9
View File
@@ -0,0 +1,9 @@
SPECIFICATION Spec
CONSTANT Ids = {s1, s2, s3}
INVARIANTS
TypeOK
ListedAreLive
NoResurrection
IndexNamesPublished
PROPERTIES
RebuildIsExact
+156
View File
@@ -0,0 +1,156 @@
----------------------------- MODULE Catalog -----------------------------
(***************************************************************************)
(* Step: L3 catalog (index.jsonl). *)
(* *)
(* The index is derived. Publishing a folder happens before its index *)
(* put; delete renames the folder to .trash before its tombstone, then *)
(* purges it. A crash can land between any two steps and forgets owed *)
(* index writes. list() filters out entries whose folder is missing and *)
(* appends a tombstone for them (Heal). rebuild() re-derives the index *)
(* from live folders and purges .trash, atomically under index.lock. *)
(* *)
(* Design: system-design.md "Concurrency And Ordering" (index) and *)
(* "Reliability And Recovery". *)
(***************************************************************************)
EXTENDS Naturals, Sequences, FiniteSets
CONSTANT Ids
VARIABLES
dir, \* [Ids -> {"none", "live", "trash", "gone"}]
index, \* a sequence of [id, op], op \in {"put", "del"}
pending, \* [Ids -> {"none", "put", "del"}]: index write still owed
everDeleted \* ghost: ids that were ever deleted
vars == <<dir, index, pending, everDeleted>>
LastOp(s) ==
LET is == {i \in 1..Len(index) : index[i].id = s} IN
IF is = {} THEN "none"
ELSE index[CHOOSE i \in is : \A j \in is : j <= i].op
Indexed == {s \in Ids : LastOp(s) = "put"}
Live == {s \in Ids : dir[s] = "live"}
\* What list() returns: indexed entries whose folder exists.
Listed == {s \in Indexed : dir[s] = "live"}
TypeOK ==
/\ dir \in [Ids -> {"none", "live", "trash", "gone"}]
/\ pending \in [Ids -> {"none", "put", "del"}]
/\ everDeleted \subseteq Ids
Init ==
/\ dir = [s \in Ids |-> "none"]
/\ index = <<>>
/\ pending = [s \in Ids |-> "none"]
/\ everDeleted = {}
\* First save: rename .tmp/{id} to {id}; the index put is owed next.
Publish(s) ==
/\ dir[s] = "none"
/\ dir' = [dir EXCEPT ![s] = "live"]
/\ pending' = [pending EXCEPT ![s] = "put"]
/\ UNCHANGED <<index, everDeleted>>
IndexPut(s) ==
/\ pending[s] = "put"
/\ index' = Append(index, [id |-> s, op |-> "put"])
/\ pending' = [pending EXCEPT ![s] = "none"]
/\ UNCHANGED <<dir, everDeleted>>
\* delete(): rename {id} to .trash/{id}; the tombstone is owed next.
Trash(s) ==
/\ dir[s] = "live"
/\ pending[s] = "none"
/\ dir' = [dir EXCEPT ![s] = "trash"]
/\ pending' = [pending EXCEPT ![s] = "del"]
/\ everDeleted' = everDeleted \cup {s}
/\ UNCHANGED index
IndexDel(s) ==
/\ pending[s] = "del"
/\ index' = Append(index, [id |-> s, op |-> "del"])
/\ pending' = [pending EXCEPT ![s] = "none"]
/\ UNCHANGED <<dir, everDeleted>>
Purge(s) ==
/\ dir[s] = "trash"
/\ pending[s] = "none"
/\ dir' = [dir EXCEPT ![s] = "gone"]
/\ UNCHANGED <<index, pending, everDeleted>>
\* A resume, or a title, workspace or language change, rewrites the entry of
\* a live session (D42). The session holds its writer lock throughout, and
\* delete needs that lock (Busy), so no Trash lands inside it. Over an entry
\* already put, a refresh changes no last op, so the spec leaves that case
\* out; over a missing one, such as a publish put a crash forgot, it heals.
Refresh(s) ==
/\ dir[s] = "live"
/\ LastOp(s) # "put"
/\ index' = Append(index, [id |-> s, op |-> "put"])
/\ UNCHANGED <<dir, pending, everDeleted>>
\* A crash forgets every owed index write.
Crash ==
/\ \E s \in Ids : pending[s] # "none"
/\ pending' = [s \in Ids |-> "none"]
/\ UNCHANGED <<dir, index, everDeleted>>
\* list() self-heal: tombstone an indexed id whose folder is missing.
Heal(s) ==
/\ s \in Indexed
/\ dir[s] # "live"
/\ index' = Append(index, [id |-> s, op |-> "del"])
/\ UNCHANGED <<dir, pending, everDeleted>>
\* rebuild(): a fresh index with one put per live folder, then one del per
\* tombstoned id, each group in any order; .trash is purged. Tombstones
\* survive so a deleted id can never be re-imported or listed again. Owed
\* writes are dropped because the fresh index already reflects the folders.
Tombstoned == {s \in Ids : LastOp(s) = "del"}
Rebuild ==
/\ LET n == Cardinality(Live)
m == Cardinality(Tombstoned)
IN \E order \in [1..n -> Live], gone \in [1..m -> Tombstoned] :
/\ \A i, j \in 1..n : i # j => order[i] # order[j]
/\ \A i, j \in 1..m : i # j => gone[i] # gone[j]
/\ index' = [i \in 1..n |-> [id |-> order[i], op |-> "put"]]
\o [i \in 1..m |-> [id |-> gone[i], op |-> "del"]]
/\ dir' = [s \in Ids |-> IF dir[s] = "trash" THEN "gone" ELSE dir[s]]
/\ pending' = [s \in Ids |-> "none"]
/\ UNCHANGED everDeleted
Next ==
\/ \E s \in Ids :
\/ Publish(s)
\/ IndexPut(s)
\/ Refresh(s)
\/ Trash(s)
\/ IndexDel(s)
\/ Purge(s)
\/ Heal(s)
\/ Crash
\/ Rebuild
Spec == Init /\ [][Next]_vars
-----------------------------------------------------------------------------
(* Invariants *)
\* list() never returns a session whose folder is gone.
ListedAreLive == Listed \subseteq Live
\* A deleted id never comes back, in the list or on disk.
NoResurrection ==
\A s \in everDeleted : s \notin Listed /\ dir[s] # "live"
\* The index only ever names ids that were published.
IndexNamesPublished ==
\A i \in 1..Len(index) : dir[index[i].id] # "none"
(* Action property: right after rebuild(), the index lists exactly the *)
(* live folders. *)
RebuildIsExact == [][Rebuild => (Indexed' = Live')]_vars
=============================================================================
+12
View File
@@ -0,0 +1,12 @@
SPECIFICATION Spec
CONSTANTS
Ids = {s1, s2, s3}
Blobs = {b1, b2}
MaxLen = 4
NoBlob = NoBlob
INVARIANTS
TypeOK
RefsExist
ForkKeepsPrefix
ForkAtBoundary
FirstLine
+159
View File
@@ -0,0 +1,159 @@
------------------------------- MODULE Fork -------------------------------
(***************************************************************************)
(* Step: L2 fork and blobs. *)
(* *)
(* A blob file lands before any line can reference it. A fork copies *)
(* lines 2..S of its source at a turn boundary and hard-links the *)
(* source's blobs. Deleting a session removes its folder, log and links. *)
(* Invariants: every blob a live log references exists in that session, *)
(* unless the disk lost it there, and a fork keeps its copied prefix *)
(* whatever later happens to its source. A lost blob damages only its *)
(* session: a fork never copies a line that names one (D39). *)
(* *)
(* Design: system-design.md D5, D6 and D39. *)
(***************************************************************************)
EXTENDS Naturals, Sequences, FiniteSets
CONSTANTS Ids, Blobs, MaxLen, NoBlob
VARIABLES
st, \* [Ids -> {"none", "live", "deleted"}]
lg, \* [Ids -> Seq(entry)]: each session's log
links, \* [Ids -> SUBSET Blobs]: files present in {id}/blobs/
base, \* [Ids -> Seq(entry)]: prefix copied at fork time (<<>> for roots)
lost \* [Ids -> SUBSET Blobs]: blob files the disk lost in that session
vars == <<st, lg, links, base, lost>>
E(k, b) == [k |-> k, b |-> b]
Count(l, k) == Cardinality({i \in 1..Len(l) : l[i].k = k})
IsOpen(l) == Count(l, "start") > Count(l, "commit")
TypeOK ==
/\ st \in [Ids -> {"none", "live", "deleted"}]
/\ links \in [Ids -> SUBSET Blobs]
/\ lost \in [Ids -> SUBSET Blobs]
Init ==
/\ st = [s \in Ids |-> "none"]
/\ lg = [s \in Ids |-> <<>>]
/\ links = [s \in Ids |-> {}]
/\ base = [s \in Ids |-> <<>>]
/\ lost = [s \in Ids |-> {}]
Create(s) ==
/\ st[s] = "none"
/\ st' = [st EXCEPT ![s] = "live"]
/\ lg' = [lg EXCEPT ![s] = <<E("created", NoBlob)>>]
/\ UNCHANGED <<links, base, lost>>
Start(s) ==
/\ st[s] = "live"
/\ Len(lg[s]) < MaxLen
/\ ~IsOpen(lg[s])
/\ lg' = [lg EXCEPT ![s] = Append(@, E("start", NoBlob))]
/\ UNCHANGED <<st, links, base, lost>>
Commit(s) ==
/\ st[s] = "live"
/\ Len(lg[s]) < MaxLen
/\ IsOpen(lg[s])
/\ lg' = [lg EXCEPT ![s] = Append(@, E("commit", NoBlob))]
/\ UNCHANGED <<st, links, base, lost>>
\* put_blob(): the file is installed before any line can reference it.
PutBlob(s, b) ==
/\ st[s] = "live"
/\ b \notin links[s]
/\ links' = [links EXCEPT ![s] = @ \cup {b}]
/\ UNCHANGED <<st, lg, base, lost>>
\* append(item{blob}): the manager checks the blob exists in this session.
Item(s, b) ==
/\ st[s] = "live"
/\ Len(lg[s]) < MaxLen
/\ IsOpen(lg[s])
/\ b \in links[s]
/\ lg' = [lg EXCEPT ![s] = Append(@, E("item", b))]
/\ UNCHANGED <<st, links, base, lost>>
\* append(set{blobs}): a setting may list blobs, inside a turn or outside it,
\* under the item's rule: the manager checks the blob exists in this
\* session. fx writes such lines when it moves a session off its side folder
\* (D47) and when its compactor keeps records (D50). One per session bounds
\* the states; more such lines behave as more items do.
SetBlob(s, b) ==
/\ st[s] = "live"
/\ Len(lg[s]) < MaxLen
/\ Count(lg[s], "set") = 0
/\ b \in links[s]
/\ lg' = [lg EXCEPT ![s] = Append(@, E("set", b))]
/\ UNCHANGED <<st, links, base, lost>>
\* open(.fork{src, at}): only at a turn boundary, and only over lines
\* whose blobs the source still has (`.last_good` ends before the first
\* that names a lost one, D39). Line 1 is new; lines 2..at are copied
\* unchanged; every source blob is hard-linked.
Fork(src, f, at) ==
/\ st[src] = "live"
/\ st[f] = "none"
/\ at \in 1..Len(lg[src])
/\ ~IsOpen(SubSeq(lg[src], 1, at))
/\ \A i \in 2..at : lg[src][i].b # NoBlob => lg[src][i].b \in links[src]
/\ LET copied == SubSeq(lg[src], 2, at) IN
/\ lg' = [lg EXCEPT ![f] = <<E("created", NoBlob)>> \o copied]
/\ base' = [base EXCEPT ![f] = copied]
/\ links' = [links EXCEPT ![f] = links[src]]
/\ st' = [st EXCEPT ![f] = "live"]
/\ UNCHANGED lost
\* delete(): the folder, its log and its blob links go away.
Delete(s) ==
/\ st[s] = "live"
/\ st' = [st EXCEPT ![s] = "deleted"]
/\ lg' = [lg EXCEPT ![s] = <<>>]
/\ links' = [links EXCEPT ![s] = {}]
/\ lost' = [lost EXCEPT ![s] = {}]
/\ UNCHANGED base
\* The disk loses one blob file of a live session (D39).
Lose(s, b) ==
/\ st[s] = "live"
/\ b \in links[s]
/\ links' = [links EXCEPT ![s] = @ \ {b}]
/\ lost' = [lost EXCEPT ![s] = @ \cup {b}]
/\ UNCHANGED <<st, lg, base>>
Next ==
\E s \in Ids :
\/ Create(s)
\/ Start(s)
\/ Commit(s)
\/ Delete(s)
\/ \E b \in Blobs : PutBlob(s, b) \/ Item(s, b) \/ SetBlob(s, b) \/ Lose(s, b)
\/ \E f \in Ids, at \in 1..MaxLen : Fork(s, f, at)
Spec == Init /\ [][Next]_vars
-----------------------------------------------------------------------------
(* Invariants *)
\* Every blob a live log references is present in that session, unless the
\* disk lost it there: a fork never inherits a missing reference.
RefsExist ==
\A s \in Ids : st[s] = "live" =>
\A i \in 1..Len(lg[s]) :
lg[s][i].b # NoBlob => lg[s][i].b \in links[s] \cup lost[s]
\* A fork's copied prefix is intact, whatever happened to its source.
ForkKeepsPrefix ==
\A f \in Ids : st[f] = "live" =>
SubSeq(lg[f], 2, Len(base[f]) + 1) = base[f]
\* A fork never starts inside an open turn.
ForkAtBoundary == \A f \in Ids : ~IsOpen(base[f])
\* Line 1 of every live log is `created`.
FirstLine == \A s \in Ids : st[s] = "live" => lg[s][1].k = "created"
=============================================================================
@@ -0,0 +1,7 @@
SPECIFICATION Spec
CONSTANT Ids = {s1, s2}
INVARIANTS
TypeOK
VisibleHasLine1
AckedIsVisible
NothingBeforeFirstTurn
+135
View File
@@ -0,0 +1,135 @@
---------------------------- MODULE Lifecycle ----------------------------
(***************************************************************************)
(* Step: L2 lazy creation and publish. *)
(* *)
(* open(.new) touches nothing on disk. The first turn builds .tmp/{id}: *)
(* mkdir, write line 1, fsync the log, rename to {id}, fsync the parent *)
(* directory, and only then returns to the caller. An fx crash stops *)
(* the sequence anywhere. A machine crash can also undo an unsynced rename,*)
(* drop an unsynced mkdir, and drop line 1 if it was not fsynced. *)
(* Rebuild sweeps leftovers in .tmp. *)
(* *)
(* Design: system-design.md D2 and "Durable State And Schema" (publish). *)
(***************************************************************************)
EXTENDS Naturals
CONSTANT Ids
Steps == {"idle", "mem", "tmpdir", "written", "synced", "renamed", "done", "dead"}
VARIABLES
step, \* [Ids -> Steps]: how far the first-turn publish has got
where, \* [Ids -> {"none", "tmp", "visible"}]: the folder on disk
line1, \* [Ids -> {"none", "cached", "durable"}]: line 1 on disk
acked \* [Ids -> BOOLEAN]: the caller was told the save succeeded
vars == <<step, where, line1, acked>>
TypeOK ==
/\ step \in [Ids -> Steps]
/\ where \in [Ids -> {"none", "tmp", "visible"}]
/\ line1 \in [Ids -> {"none", "cached", "durable"}]
/\ acked \in [Ids -> BOOLEAN]
Init ==
/\ step = [s \in Ids |-> "idle"]
/\ where = [s \in Ids |-> "none"]
/\ line1 = [s \in Ids |-> "none"]
/\ acked = [s \in Ids |-> FALSE]
Advance(s, from, to) ==
/\ step[s] = from
/\ step' = [step EXCEPT ![s] = to]
OpenNew(s) ==
/\ Advance(s, "idle", "mem")
/\ UNCHANGED <<where, line1, acked>>
MkTmp(s) ==
/\ Advance(s, "mem", "tmpdir")
/\ where' = [where EXCEPT ![s] = "tmp"]
/\ UNCHANGED <<line1, acked>>
WriteLine1(s) ==
/\ Advance(s, "tmpdir", "written")
/\ line1' = [line1 EXCEPT ![s] = "cached"]
/\ UNCHANGED <<where, acked>>
FsyncLog(s) ==
/\ Advance(s, "written", "synced")
/\ line1' = [line1 EXCEPT ![s] = "durable"]
/\ UNCHANGED <<where, acked>>
Rename(s) ==
/\ Advance(s, "synced", "renamed")
/\ where' = [where EXCEPT ![s] = "visible"]
/\ UNCHANGED <<line1, acked>>
\* fsync of sessions/v2/ makes the rename durable; then append() returns.
FsyncDirAndAck(s) ==
/\ Advance(s, "renamed", "done")
/\ acked' = [acked EXCEPT ![s] = TRUE]
/\ UNCHANGED <<where, line1>>
\* fx dies: every unfinished first-turn publish stops; the disk keeps what it has.
ProcessCrash ==
/\ \E s \in Ids : step[s] \notin {"idle", "done", "dead"}
/\ step' = [s \in Ids |-> IF step[s] \in {"idle", "done"} THEN step[s] ELSE "dead"]
/\ UNCHANGED <<where, line1, acked>>
\* Machine crash (the PowerLoss action): like an fx crash, and unsynced directory changes and unsynced
\* line 1 bytes may also be lost. An unsynced rename may be undone, and when
\* the mkdir of .tmp/{id} was never synced either, the folder may be gone
\* from both places. Unsynced line 1 bytes may vanish or survive; after the
\* reboot, whatever survived is durable.
PowerLoss ==
\E lose \in [Ids -> BOOLEAN], gone \in [Ids -> BOOLEAN], keep \in [Ids -> BOOLEAN] :
LET w == [s \in Ids |->
CASE step[s] = "renamed" /\ lose[s] -> IF gone[s] THEN "none" ELSE "tmp"
[] where[s] = "tmp" /\ step[s] # "done" /\ lose[s] -> "none"
[] OTHER -> where[s]]
IN /\ where' = w
/\ line1' = [s \in Ids |->
CASE w[s] = "none" -> "none"
[] line1[s] = "cached" /\ keep[s] -> "durable"
[] line1[s] = "cached" -> "none"
[] OTHER -> line1[s]]
/\ step' = [s \in Ids |->
IF step[s] \in {"idle", "done"} THEN step[s] ELSE "dead"]
/\ UNCHANGED acked
\* rebuild(): remove a leftover .tmp folder of a first-turn publish that died.
Sweep(s) ==
/\ step[s] = "dead"
/\ where[s] = "tmp"
/\ where' = [where EXCEPT ![s] = "none"]
/\ line1' = [line1 EXCEPT ![s] = "none"]
/\ UNCHANGED <<step, acked>>
Next ==
\/ \E s \in Ids :
\/ OpenNew(s)
\/ MkTmp(s)
\/ WriteLine1(s)
\/ FsyncLog(s)
\/ Rename(s)
\/ FsyncDirAndAck(s)
\/ Sweep(s)
\/ ProcessCrash
\/ PowerLoss
Spec == Init /\ [][Next]_vars
-----------------------------------------------------------------------------
(* Invariants *)
\* A visible session always has a durable first line.
VisibleHasLine1 == \A s \in Ids : where[s] = "visible" => line1[s] = "durable"
\* An acknowledged first turn stays visible, even after a machine crash.
AckedIsVisible == \A s \in Ids : acked[s] => where[s] = "visible"
\* open(.new) and settings held before the first turn never create anything on disk.
NothingBeforeFirstTurn ==
\A s \in Ids : step[s] \in {"idle", "mem"} => where[s] = "none"
=============================================================================
@@ -0,0 +1,9 @@
SPECIFICATION FairSpec
CONSTANT Handles = {h1, h2, h3}
INVARIANTS
TypeOK
Resolvable
TerminalKept
MapComplete
PROPERTIES
EventuallyDone
@@ -0,0 +1,126 @@
--------------------------- MODULE MoveSideFiles ---------------------------
(***************************************************************************)
(* Step: fx wiring, moving an older v2 session off its side folder (D47). *)
(* *)
(* An older v2 session keeps large bodies and its terminal state in *)
(* ~/.fx/session-files/{id}/. On its first writable open fx puts each *)
(* body as a blob, writes one batch holding the map from old handles to *)
(* blobs and the two settings, renames the terminal folder to *)
(* ~/.fx/terminal/{id}/, and removes the side folder. One writer at a *)
(* time holds the session (WriterLock), and a crash can stop it between *)
(* any two steps; the next open starts again from what the disk holds. *)
(* A reader may resolve any old handle at any moment: through the map *)
(* once it is written, through the side folder before. *)
(* *)
(* Design: system-design.md D44 to D48. *)
(***************************************************************************)
EXTENDS Naturals, FiniteSets
CONSTANTS Handles
VARIABLES
side, \* SUBSET Handles: bodies still in the side folder
folder, \* "present" | "removed": the side folder itself
blobs, \* SUBSET Handles: bodies put as blobs of the session
mapped, \* TRUE once the batch with the map and settings is durable
terminal, \* "side" | "moved": where the terminal state lives
open \* TRUE while a writer is running the move
vars == <<side, folder, blobs, mapped, terminal, open>>
TypeOK ==
/\ side \subseteq Handles
/\ folder \in {"present", "removed"}
/\ blobs \subseteq Handles
/\ mapped \in BOOLEAN
/\ terminal \in {"side", "moved"}
/\ open \in BOOLEAN
Init ==
/\ side = Handles
/\ folder = "present"
/\ blobs = {}
/\ mapped = FALSE
/\ terminal = "side"
/\ open = FALSE
\* A writable open takes the session; the move runs inside it.
Open ==
/\ ~open
/\ open' = TRUE
/\ UNCHANGED <<side, folder, blobs, mapped, terminal>>
\* put_blob() of one side body; repeating it is harmless (content addressed).
PutBlob(h) ==
/\ open
/\ ~mapped
/\ h \in side
/\ blobs' = blobs \cup {h}
/\ UNCHANGED <<side, folder, mapped, terminal, open>>
\* One durable batch: `set moved_files` listing every blob, and the settings.
WriteMap ==
/\ open
/\ ~mapped
/\ blobs = Handles
/\ mapped' = TRUE
/\ UNCHANGED <<side, folder, blobs, terminal, open>>
\* Rename session-files/{id}/terminal to terminal/{id}.
MoveTerminal ==
/\ open
/\ mapped
/\ terminal = "side"
/\ terminal' = "moved"
/\ UNCHANGED <<side, folder, blobs, mapped, open>>
\* Remove the side folder and everything left in it.
RemoveSide ==
/\ open
/\ mapped
/\ terminal = "moved"
/\ folder = "present"
/\ folder' = "removed"
/\ side' = {}
/\ UNCHANGED <<blobs, mapped, terminal, open>>
\* The writer stops: a clean close or a crash between any two steps.
Stop ==
/\ open
/\ open' = FALSE
/\ UNCHANGED <<side, folder, blobs, mapped, terminal>>
Done == folder = "removed"
Next ==
\/ Open
\/ \E h \in Handles : PutBlob(h)
\/ WriteMap
\/ MoveTerminal
\/ RemoveSide
\/ Stop
Spec == Init /\ [][Next]_vars
\* Writers keep coming back, and a step enabled again and again eventually
\* runs (a crash can stop any one open, not every open).
FairSpec == Spec /\ WF_vars(Open) /\ SF_vars(WriteMap) /\ SF_vars(MoveTerminal)
/\ SF_vars(RemoveSide) /\ \A h \in Handles : SF_vars(PutBlob(h))
-----------------------------------------------------------------------------
(* Invariants *)
\* Every old handle resolves at every moment: through the map once it is
\* written, through the side folder before.
Resolvable ==
\A h \in Handles : (mapped /\ h \in blobs) \/ (~mapped /\ h \in side)
\* The terminal state is never lost: the side folder goes only after it moved.
TerminalKept == folder = "removed" => terminal = "moved"
\* The map names only bodies that exist as blobs.
MapComplete == mapped => blobs = Handles
\* Liveness under FairSpec: the move eventually finishes and stays finished.
EventuallyDone == <>[]Done
=============================================================================
+148
View File
@@ -0,0 +1,148 @@
# TLA+ models
One model per step of the design. Each one checks the protocol described in
`../system-design.md`, not the Zig code. The bridge to the code is that every
invariant below also becomes a test assertion for the step it models.
**Status: all eight specs parse (SANY) and pass TLC. Each one also catches
its planted bugs (`./mutants.py`).** Last run: 2026-09-27, TLC2 2.19 (rev
5a47802), OpenJDK 27 on macOS arm64.
| Spec | States generated | Distinct states | Search depth | Result |
|---|---|---|---|---|
| `SessionLog` | 686 | 110 | 15 | No error |
| `TurnLifecycle` | 21,684 | 19,352 | 14 | No error |
| `WriterLock` | 3,257 | 1,016 | 24 | No error |
| `ResumeSnapshot` | 5,461 | 5,461 | 7 | No error |
| `Fork` | 375,727 | 51,776 | 16 | No error |
| `Subagents` | 204,097 | 120,297 | 23 | No error |
| `Catalog` | 645,269 | 94,482 | 19 | No error |
| `Lifecycle` | 9,553 | 144 | 13 | No error |
Every run explored the whole bounded state space (0 states left on the queue).
`Lifecycle`'s `PowerLoss` was widened on 2026-09-27, when trace validation
compared it with real disk behavior. A renamed but unsynced publish may also
vanish from both folders (when the `mkdir` of `.tmp/{id}` was never synced
either), and unsynced line 1 bytes may survive as well as vanish. The spec
still passes, and both of its mutants are still caught.
`Catalog`'s `Rebuild` was widened the same day to keep a `del` for every
tombstoned id after its fresh `put` lines, as the code does, so a deleted id
stays deleted through a rebuild and cannot be re-imported. The spec still
passes, and its mutant is still caught.
`Subagents` was widened on 2026-09-27 for D22 (the pre-wiring API check):
work numbers per child, at most one unfinished work item per child, `failed`
finishes and a parent `cancelled`. `LostMeansNoLog` became an action
property, checked on the step that writes the line, because a persistent
child marked lost for one work item may create its log with a later one.
`./mutants.py` plants 14 realistic bugs, one to three per spec, and requires
TLC to report the matching invariant or action property as violated. All 14
are caught:
- fsync acknowledging a half-written line
- open() keeping a torn tail
- reopen skipping `turn_interrupted`
- accepting an item outside a turn
- open ignoring the flock
- a snapshot with the wrong state
- a fork forgetting blob links
- a fork inside an open turn
- repair marking every child lost
- a spawn while the child's work is open
- reopen skipping child repair
- list skipping the folder check
- rename before the line 1 fsync
- ack before the directory fsync
## The specs
| Spec | Step (level) | What it proves | Invariants |
|---|---|---|---|
| `SessionLog.tla` | Log append, fsync, fx crash, machine crash, torn tail (L1) | Acknowledged lines survive a machine crash; only the final line can be torn; open() leaves no torn line | `SeqContiguous`, `OnlyTailTorn`, `AckedSurvive`, `OpenIsClean` |
| `TurnLifecycle.tla` | Turn rules, close, crash, reopen (L2) | The manager only writes valid turn transitions; close and reopen never leave a stale open turn; the log is only ever extended | `StartsWithCreated`, `AtMostOneOpenTurn`, `ItemsInsideTurns`, `EndsCloseATurn`, `ClosedOnlyBetweenTurns`, `NoStaleTurnAfterReopen`, property `AppendOnly` |
| `WriterLock.tla` | flock across processes, Session mutex across threads (L0, L2) | One writing process; file order equals seq order across threads and after takeover | `OneWriter`, `OnlyHolderOpen`, `MutexInsideOpenSession`, `SeqIsPosition` |
| `ResumeSnapshot.tla` | Fold and resume from a snapshot (L2) | Resuming from the newest snapshot equals a full replay; snapshots can be dropped | `ResumeEqualsFullReplay`, `SnapshotsAreIgnorable` |
| `Lifecycle.tla` | Lazy creation and publish (L2) | Nothing on disk before the first turn; a visible session has a durable line 1; an acknowledged first turn survives a machine crash | `VisibleHasLine1`, `AckedIsVisible`, `NothingBeforeFirstTurn` |
| `Fork.tla` | Fork and blobs (L2) | Every referenced blob exists; a fork keeps its prefix after the source changes or is deleted; forks start at turn boundaries | `RefsExist`, `ForkKeepsPrefix`, `ForkAtBoundary`, `FirstLine` |
| `Subagents.tla` | Children, persistent work, cancel, parent crash and repair (L2) | Every work item ends in a known state; one unfinished work item per child; `lost` only without a log; no orphan child logs | `OneOpenWork`, `FinishClosesOpenWork`, `NoOrphanChild`, `NoIndeterminate`, property `LostMeansNoLog` |
| `Catalog.tla` | index.jsonl, list self-heal, delete, rebuild (L3) | list never returns a missing session; a deleted id never comes back; rebuild is exact | `ListedAreLive`, `NoResurrection`, `IndexNamesPublished`, property `RebuildIsExact` |
Every spec also checks `TypeOK` where one is defined.
**What the `PowerLoss` action means.** It models a machine crash: the OS
goes down without flushing the page cache, and only synced bytes survive.
With D3's plain `fsync`, that covers:
- an OS crash (kernel panic or freeze) on every platform;
- a power cut on Linux.
A power cut on macOS can also lose recently synced data from the drive's own
cache. D3 accepts that, and it is outside these models. The action keeps its
name so the trace event contract and the mutants stay stable.
## Running
```sh
./check.sh # all specs
./check.sh SessionLog Fork # some specs
./mutants.py # every planted bug must still be caught
```
Requirements:
- OpenJDK: `brew install openjdk`. It is keg-only, so it is not on PATH;
both scripts find it at `/opt/homebrew/opt/openjdk/bin/java`.
- `tla2tools.jar`: `~/.local/share/tla/tla2tools.jar`. Override the location
with `TLA2TOOLS_JAR`.
`check.sh` runs TLC with `-deadlock`, because every model is bounded and stops
at its bound, and it keeps TLC's state files outside this folder. When you add
an invariant, add a mutant to `mutants.py` that breaks it.
## Checking the code, not just the design
The models above check the design. To check that the Zig code actually
behaves like them, the code writes a trace of its steps in test builds, and
TLC validates that trace against the same specs. See
[trace/README.md](trace/README.md).
```sh
./trace/validate.sh SessionLog run.ndjson # ACCEPTED, or REJECTED at step N
./trace/selftest.sh # the validator must accept and reject
```
## What the models do not cover
- Byte encoding, JSON parsing and line size caps. Those are covered by unit
tests.
- Byte offsets. `Fork.tla` copies lines as values, so it cannot see that a
snapshot's `compaction_offset` moves when the fork's line 1 is longer; the
driver found that, and the fork now encodes each copied snapshot again
(`session.zig` test "a fork after a compaction and snapshots").
- Which bytes open reads. `SessionLog.tla` recovers a whole log; the manager
reads only line 1, the newest snapshot and what follows, so older damage
is found by `read` and `verify` (`system-design.md` Reliability And
Recovery; the driver's bit-flip cells cover every region).
- Code paths that no test runs. Trace validation (above) covers only the
runs we execute.
- Performance. That is the benchmark in `../system-design.md` Performance
And Capacity.
- The fx adapter and host policy, such as when to offer "continue". Those are
covered by fx E2E tests.
- Liveness (for example "every child eventually finishes"). The models check
safety only. Liveness would need fairness assumptions about the model
provider, which the manager does not control.
- Model bounds. Small bounds (2 or 3 ids, logs of 4 to 7 lines) find ordering
and crash-window bugs, but they do not prove behavior at every size. Raise
a bound in a spec's `.cfg` when a bug is suspected past it.
## Keeping the models honest
When a decision in `../system-design.md` changes, update the spec that models
it in the same change. If TLC finds a counterexample:
1. If the spec is wrong, fix the spec.
2. If the design is wrong, fix `system-design.md` first, then the spec.
@@ -0,0 +1,8 @@
SPECIFICATION Spec
CONSTANTS
Vals = {v1, v2}
MaxLen = 6
NoVal = NoVal
INVARIANTS
ResumeEqualsFullReplay
SnapshotsAreIgnorable
@@ -0,0 +1,91 @@
-------------------------- MODULE ResumeSnapshot --------------------------
(***************************************************************************)
(* Step: L2 fold and resume. *)
(* *)
(* open(.resume) folds from the newest snapshot instead of line 1. This *)
(* spec states the contract that makes that safe: resuming from the *)
(* newest snapshot gives exactly the state of a full replay, and the *)
(* snapshot lines themselves can be dropped without changing any state. *)
(* The Zig implementation must pass the same two checks as property *)
(* tests. *)
(* *)
(* Design: system-design.md "Fold". *)
(***************************************************************************)
EXTENDS Naturals, Sequences
CONSTANTS Vals, MaxLen, NoVal
Empty == [pref |-> NoVal, turns |-> 0, open |-> FALSE]
\* Every line has the same record shape; `st` is used only by snapshots.
Entry(k, v, s) == [k |-> k, v |-> v, st |-> s]
Step(s, e) ==
CASE e.k = "set" -> [s EXCEPT !.pref = e.v]
[] e.k = "start" -> [s EXCEPT !.open = TRUE]
[] e.k = "commit" -> [s EXCEPT !.open = FALSE, !.turns = @ + 1]
[] OTHER -> s \* snapshot lines change nothing
RECURSIVE FoldFrom(_, _, _)
FoldFrom(l, i, s) ==
IF i > Len(l) THEN s ELSE FoldFrom(l, i + 1, Step(s, l[i]))
Full(l) == FoldFrom(l, 1, Empty)
SnapIdx(l) == {i \in 1..Len(l) : l[i].k = "snap"}
LastSnap(l) ==
IF SnapIdx(l) = {} THEN 0
ELSE CHOOSE i \in SnapIdx(l) : \A j \in SnapIdx(l) : j <= i
\* What open(.resume) computes: the newest snapshot, then fold the tail.
Resume(l) ==
LET j == LastSnap(l) IN
IF j = 0 THEN Full(l) ELSE FoldFrom(l, j + 1, l[j].st)
RECURSIVE Strip(_)
Strip(l) ==
IF l = <<>> THEN <<>>
ELSE IF Head(l).k = "snap" THEN Strip(Tail(l))
ELSE <<Head(l)>> \o Strip(Tail(l))
VARIABLES log
vars == <<log>>
Init == log = <<>>
Set(v) ==
/\ Len(log) < MaxLen
/\ log' = Append(log, Entry("set", v, Empty))
Start ==
/\ Len(log) < MaxLen
/\ ~Full(log).open
/\ log' = Append(log, Entry("start", NoVal, Empty))
Commit ==
/\ Len(log) < MaxLen
/\ Full(log).open
/\ log' = Append(log, Entry("commit", NoVal, Empty))
\* The manager writes a snapshot of the state folded so far. Snapshots
\* may land anywhere, including inside an open turn.
Snapshot ==
/\ Len(log) < MaxLen
/\ log' = Append(log, Entry("snap", NoVal, Full(log)))
Next ==
\/ \E v \in Vals : Set(v)
\/ Start
\/ Commit
\/ Snapshot
Spec == Init /\ [][Next]_vars
-----------------------------------------------------------------------------
(* Invariants *)
ResumeEqualsFullReplay == Resume(log) = Full(log)
SnapshotsAreIgnorable == Full(Strip(log)) = Full(log)
=============================================================================
@@ -0,0 +1,8 @@
SPECIFICATION Spec
CONSTANT MaxLines = 4
INVARIANTS
TypeOK
SeqContiguous
OnlyTailTorn
AckedSurvive
OpenIsClean
+122
View File
@@ -0,0 +1,122 @@
---------------------------- MODULE SessionLog ----------------------------
(***************************************************************************)
(* Step: L1 log. One append-only log file. *)
(* *)
(* Models write() into the page cache, fsync at a durability point, an fx *)
(* crash (the kernel keeps every written byte), a machine crash (only the *)
(* fsynced prefix is guaranteed; the rest may be lost and the new last *)
(* line may be torn) and the torn-tail cut plus fsync done by open(). *)
(* *)
(* Design: system-design.md "Line format" and "Durable State And Schema". *)
(***************************************************************************)
EXTENDS Naturals, Sequences
CONSTANT MaxLines
VARIABLES
log, \* file contents: a sequence of lines [seq, done]
durable, \* length of the prefix that is on stable storage
acked, \* number of lines a caller was told are durable
status \* "open", "writing" or "down"
vars == <<log, durable, acked, status>>
Line(n, d) == [seq |-> n, done |-> d]
LastDone == Len(log) = 0 \/ log[Len(log)].done
TypeOK ==
/\ log \in Seq([seq : 1..MaxLines, done : BOOLEAN])
/\ Len(log) <= MaxLines
/\ durable \in 0..MaxLines
/\ acked \in 0..MaxLines
/\ status \in {"open", "writing", "down"}
Init ==
/\ log = <<>>
/\ durable = 0
/\ acked = 0
/\ status = "open"
(* write() starts: bytes are in the page cache, the newline is not yet. *)
BeginWrite ==
/\ status = "open"
/\ Len(log) < MaxLines
/\ log' = Append(log, Line(Len(log) + 1, FALSE))
/\ status' = "writing"
/\ UNCHANGED <<durable, acked>>
(* write() returns: the whole line, including its newline, is written. *)
FinishWrite ==
/\ status = "writing"
/\ log' = [log EXCEPT ![Len(log)].done = TRUE]
/\ status' = "open"
/\ UNCHANGED <<durable, acked>>
(* Durability point: fsync, then acknowledge to the caller. *)
Fsync ==
/\ status = "open"
/\ durable' = Len(log)
/\ acked' = Len(log)
/\ UNCHANGED <<log, status>>
(* fx dies. The kernel keeps every byte already written. *)
ProcessCrash ==
/\ status \in {"open", "writing"}
/\ status' = "down"
/\ UNCHANGED <<log, durable, acked>>
(* Machine crash (OS crash, or a power cut on Linux; D3 leaves a macOS *)
(* power cut outside the model): bytes after the durable prefix may be *)
(* lost, and the new *)
(* last line may be torn. *)
PowerLoss ==
/\ \E k \in durable..Len(log), torn \in BOOLEAN :
LET kept == SubSeq(log, 1, k) IN
log' = IF torn /\ k > durable
THEN [kept EXCEPT ![k].done = FALSE]
ELSE kept
/\ status' = "down"
/\ UNCHANGED <<durable, acked>>
(* open(.resume): cut a torn last line, fsync, accept writes again. *)
Recover ==
/\ status = "down"
/\ LET repaired == IF LastDone THEN log ELSE SubSeq(log, 1, Len(log) - 1)
IN /\ log' = repaired
/\ durable' = Len(repaired)
/\ status' = "open"
/\ UNCHANGED acked
Next ==
\/ BeginWrite
\/ FinishWrite
\/ Fsync
\/ ProcessCrash
\/ PowerLoss
\/ Recover
Spec == Init /\ [][Next]_vars
-----------------------------------------------------------------------------
(* Invariants *)
\* seq equals the line's position: no gaps and no duplicates.
SeqContiguous == \A i \in 1..Len(log) : log[i].seq = i
\* Only the final line can be incomplete.
OnlyTailTorn == \A i \in 1..(Len(log) - 1) : log[i].done
\* A line the caller was told is durable is never lost or torn.
AckedSurvive ==
/\ acked <= durable
/\ durable <= Len(log)
/\ \A i \in 1..acked : log[i].done
\* An open log never ends in a torn line.
OpenIsClean == status = "open" => LastDone
(* Note: after a machine crash, a line past `acked` can vanish and its seq *)
(* is handed out again. Hosts must treat only acknowledged seqs as *)
(* stable (system-design.md "Acknowledged seq"). *)
=============================================================================
@@ -0,0 +1,11 @@
SPECIFICATION Spec
CONSTANT Children = {c1, c2}
CONSTANT MaxWork = 2
INVARIANTS
TypeOK
OneOpenWork
FinishClosesOpenWork
NoOrphanChild
NoIndeterminate
PROPERTY
LostMeansNoLog
+170
View File
@@ -0,0 +1,170 @@
---------------------------- MODULE Subagents ----------------------------
(***************************************************************************)
(* Step: L2 children. *)
(* *)
(* The parent log records child_spawned (durable) before the child's work *)
(* starts. A child's first turn creates its own log; a persistent child's *)
(* later work runs in that same log. The child thread appends *)
(* child_finished (ok or failed), or the parent cancels the work *)
(* (cancelled). A crash can land between any two steps. On parent reopen, *)
(* the manager closes every unfinished work item: `lost` if the child has *)
(* no log, `interrupted` if it has one. *)
(* *)
(* A child has at most one unfinished work item (D22); its later work *)
(* items are new spawn/finish pairs with the next work number. fx's *)
(* `data` on these lines is opaque to the manager and not modeled. *)
(* *)
(* Design: system-design.md kinds table, "Reliability And Recovery" and *)
(* D22. *)
(***************************************************************************)
EXTENDS Naturals, Sequences, FiniteSets
CONSTANTS Children, MaxWork
VARIABLES
plog, \* parent log: a sequence of [k, c, w, o]
childLog, \* [Children -> BOOLEAN]: {c}/log.jsonl exists
thread, \* [Children -> {"none", "starting", "running"}], in memory
up, \* the fx process is running
repairing \* reopen repair is in progress
vars == <<plog, childLog, thread, up, repairing>>
Outcomes == {"ok", "failed", "cancelled", "interrupted", "lost"}
Count(s, k, c) == Cardinality({i \in 1..Len(s) : s[i].k = k /\ s[i].c = c})
Works(c) == Count(plog, "spawned", c)
Unfinished(c) == Works(c) > Count(plog, "finished", c)
Spawned(c) == Works(c) > 0
TypeOK ==
/\ \A i \in 1..Len(plog) :
/\ plog[i].k \in {"spawned", "finished"}
/\ plog[i].c \in Children
/\ plog[i].w \in 1..MaxWork
/\ plog[i].o \in Outcomes \cup {"none"}
/\ childLog \in [Children -> BOOLEAN]
/\ thread \in [Children -> {"none", "starting", "running"}]
/\ up \in BOOLEAN
/\ repairing \in BOOLEAN
Init ==
/\ plog = <<>>
/\ childLog = [c \in Children |-> FALSE]
/\ thread = [c \in Children |-> "none"]
/\ up = TRUE
/\ repairing = FALSE
Normal == up /\ ~repairing
\* 1. The parent appends child_spawned (fsynced), then starts the thread.
\* Refused while the child still has unfinished work.
Spawn(c) ==
/\ Normal
/\ ~Unfinished(c)
/\ Works(c) < MaxWork
/\ plog' = Append(plog, [k |-> "spawned", c |-> c, w |-> Works(c) + 1, o |-> "none"])
/\ thread' = [thread EXCEPT ![c] = "starting"]
/\ UNCHANGED <<childLog, up, repairing>>
\* 2. The child's work starts a turn; the first one publishes its log.
ChildStart(c) ==
/\ Normal
/\ thread[c] = "starting"
/\ childLog' = [childLog EXCEPT ![c] = TRUE]
/\ thread' = [thread EXCEPT ![c] = "running"]
/\ UNCHANGED <<plog, up, repairing>>
\* 3. The child finishes and appends child_finished to the parent.
ChildFinish(c) ==
/\ Normal
/\ thread[c] = "running"
/\ \E o \in {"ok", "failed"} :
plog' = Append(plog, [k |-> "finished", c |-> c, w |-> Works(c), o |-> o])
/\ thread' = [thread EXCEPT ![c] = "none"]
/\ UNCHANGED <<childLog, up, repairing>>
\* 3'. The parent cancels the child's unfinished work and stops its thread.
Cancel(c) ==
/\ Normal
/\ thread[c] \in {"starting", "running"}
/\ plog' = Append(plog, [k |-> "finished", c |-> c, w |-> Works(c), o |-> "cancelled"])
/\ thread' = [thread EXCEPT ![c] = "none"]
/\ UNCHANGED <<childLog, up, repairing>>
\* The process dies anywhere. Durable lines stay; threads are gone.
Crash ==
/\ up
/\ up' = FALSE
/\ thread' = [c \in Children |-> "none"]
/\ repairing' = FALSE
/\ UNCHANGED <<plog, childLog>>
\* Parent reopen: repair runs before any normal work.
Reopen ==
/\ ~up
/\ up' = TRUE
/\ repairing' = TRUE
/\ UNCHANGED <<plog, childLog, thread>>
RepairChild(c) ==
/\ up
/\ repairing
/\ Unfinished(c)
/\ plog' = Append(plog, [k |-> "finished", c |-> c, w |-> Works(c),
o |-> IF childLog[c] THEN "interrupted" ELSE "lost"])
/\ UNCHANGED <<childLog, thread, up, repairing>>
RepairDone ==
/\ up
/\ repairing
/\ \A c \in Children : ~Unfinished(c)
/\ repairing' = FALSE
/\ UNCHANGED <<plog, childLog, thread, up>>
Next ==
\/ \E c \in Children :
\/ Spawn(c)
\/ ChildStart(c)
\/ ChildFinish(c)
\/ Cancel(c)
\/ RepairChild(c)
\/ Crash
\/ Reopen
\/ RepairDone
Spec == Init /\ [][Next]_vars
-----------------------------------------------------------------------------
(* Invariants *)
\* A child has at most one unfinished work item.
OneOpenWork ==
\A c \in Children :
Count(plog, "finished", c) <= Works(c) /\ Works(c) <= Count(plog, "finished", c) + 1
\* Every child_finished closes that child's newest work item, which was
\* unfinished until then: no finish before its spawn, none twice.
FinishClosesOpenWork ==
\A i \in 1..Len(plog) :
plog[i].k = "finished" =>
LET p == SubSeq(plog, 1, i - 1) IN
/\ Count(p, "spawned", plog[i].c) = Count(p, "finished", plog[i].c) + 1
/\ plog[i].w = Count(p, "spawned", plog[i].c)
\* A child log never exists without a spawn line in the parent.
NoOrphanChild == \A c \in Children : childLog[c] => Spawned(c)
\* `lost` is recorded only when the child really has no log at that moment.
\* An action property: a persistent child marked lost for one work item may
\* create its log with a later one.
LostMeansNoLog ==
[][\A i \in (Len(plog) + 1)..Len(plog') :
(plog'[i].k = "finished" /\ plog'[i].o = "lost") => ~childLog[plog'[i].c]]_vars
\* Outside repair, every unfinished work item has a live thread: no child
\* is ever left in an unknown state.
NoIndeterminate ==
Normal => \A c \in Children :
Unfinished(c) => thread[c] \in {"starting", "running"}
=============================================================================
@@ -0,0 +1,12 @@
SPECIFICATION Spec
CONSTANT MaxLen = 7
INVARIANTS
TypeOK
StartsWithCreated
AtMostOneOpenTurn
ItemsInsideTurns
EndsCloseATurn
ClosedOnlyBetweenTurns
NoStaleTurnAfterReopen
PROPERTIES
AppendOnly
@@ -0,0 +1,127 @@
--------------------------- MODULE TurnLifecycle ---------------------------
(***************************************************************************)
(* Step: L2 turn rules, close, crash and reopen repair. *)
(* *)
(* A host may ask for any kind in any order. The manager appends only *)
(* valid transitions and rejects the rest (InvalidTransition). close() *)
(* interrupts an open turn before `closed`. open(.resume) interrupts a *)
(* turn left open by a crash. Byte-level durability is SessionLog.tla's *)
(* job; here a crash keeps the log exactly as written. Every interrupt *)
(* reason (host: cancel, failed; manager: closed, crash) is one *)
(* "interrupt": the turn rules do not depend on it (D21). *)
(* *)
(* Design: system-design.md kinds table and "Fold". *)
(***************************************************************************)
EXTENDS Naturals, Sequences, FiniteSets
CONSTANT MaxLen
HostKinds == {"start", "item", "compacted", "commit", "interrupt", "set"}
Kinds == HostKinds \cup {"created", "closed"}
VARIABLES
log, \* the session log as a sequence of kinds
status, \* "new", "open", "closed" or "down"
fresh \* TRUE right after open(.resume), until the next turn starts
vars == <<log, status, fresh>>
Count(l, k) == Cardinality({i \in 1..Len(l) : l[i] = k})
Ends(l) == Count(l, "commit") + Count(l, "interrupt")
IsOpen(l) == Count(l, "start") > Ends(l)
Prefix(a, b) == Len(a) <= Len(b) /\ SubSeq(b, 1, Len(a)) = a
\* What the manager accepts from a host.
Valid(l, k) ==
CASE k = "start" -> ~IsOpen(l)
[] k = "item" -> IsOpen(l)
[] k = "commit" -> IsOpen(l)
[] k = "interrupt" -> IsOpen(l)
[] k = "compacted" -> TRUE
[] k = "set" -> TRUE
TypeOK ==
/\ log \in Seq(Kinds)
/\ status \in {"new", "open", "closed", "down"}
/\ fresh \in BOOLEAN
Init ==
/\ log = <<>>
/\ status = "new"
/\ fresh = FALSE
\* The first turn publishes the session with line 1 (D2).
Create ==
/\ status = "new"
/\ log' = <<"created">>
/\ status' = "open"
/\ UNCHANGED fresh
\* A host asks to append k; only valid kinds are written.
Write(k) ==
/\ status = "open"
/\ Len(log) < MaxLen
/\ Valid(log, k)
/\ log' = Append(log, k)
/\ fresh' = (fresh /\ k # "start")
/\ UNCHANGED status
\* close(): an open turn is interrupted, then `closed` is appended.
Close ==
/\ status = "open"
/\ Len(log) + 2 <= MaxLen
/\ log' = IF IsOpen(log)
THEN log \o <<"interrupt", "closed">>
ELSE Append(log, "closed")
/\ status' = "closed"
/\ fresh' = FALSE
\* fx dies with the session open.
Crash ==
/\ status = "open"
/\ status' = "down"
/\ UNCHANGED <<log, fresh>>
\* open(.resume): a turn left open by a crash gets turn_interrupted.
Reopen ==
/\ status \in {"closed", "down"}
/\ Len(log) < MaxLen
/\ log' = IF IsOpen(log) THEN Append(log, "interrupt") ELSE log
/\ status' = "open"
/\ fresh' = TRUE
Next ==
\/ Create
\/ \E k \in HostKinds : Write(k)
\/ Close
\/ Crash
\/ Reopen
Spec == Init /\ [][Next]_vars
-----------------------------------------------------------------------------
(* Invariants *)
StartsWithCreated == Len(log) > 0 => log[1] = "created"
AtMostOneOpenTurn == Count(log, "start") - Ends(log) \in {0, 1}
ItemsInsideTurns ==
\A i \in 1..Len(log) :
log[i] = "item" => IsOpen(SubSeq(log, 1, i - 1))
EndsCloseATurn ==
\A i \in 1..Len(log) :
log[i] \in {"commit", "interrupt"} => IsOpen(SubSeq(log, 1, i - 1))
ClosedOnlyBetweenTurns ==
\A i \in 1..Len(log) :
log[i] = "closed" => ~IsOpen(SubSeq(log, 1, i - 1))
\* After open(.resume), no turn from the previous owner is still open.
NoStaleTurnAfterReopen == fresh => ~IsOpen(log)
(* Action property: the log is only ever extended, never rewritten. *)
AppendOnly == [][Prefix(log, log')]_vars
=============================================================================
@@ -0,0 +1,36 @@
SPECIFICATION FairSpec
CONSTANTS
Procs = {p1, p2}
RootId = RootId
ChildId = ChildId
NewIds = {n}
None = None
Torn = Torn
MaxV = 2
MaxLocks = 2
MaxExtra = 1
MaxCrash = 2
MaxReq = 3
AvoidV1Ids = TRUE
INVARIANTS
TypeOK
LockHeld
NoStaleConvLock
CopyIsCurrent
NoTornImport
NoShadow
ChildrenLanded
OrphansKeepV1
UsageKept
NoDanglingMark
PROPERTIES
AnsweredWhole
NoFalseNotFound
NothingHalfConverted
NoLossOnDelete
LeftoversGone
ConverterOnlyDeletes
ConvertsOnlyOnRequest
RootsStick
ChildrenStick
RequestsAnswered
@@ -0,0 +1,36 @@
SPECIFICATION FairSpec
CONSTANTS
Procs = {p1, p2}
RootId = RootId
ChildId = ChildId
NewIds = {n}
None = None
Torn = Torn
MaxV = 2
MaxLocks = 1
MaxExtra = 1
MaxCrash = 1
MaxReq = 2
AvoidV1Ids = TRUE
INVARIANTS
TypeOK
LockHeld
NoStaleConvLock
CopyIsCurrent
NoTornImport
NoShadow
ChildrenLanded
OrphansKeepV1
UsageKept
NoDanglingMark
PROPERTIES
AnsweredWhole
NoFalseNotFound
NothingHalfConverted
NoLossOnDelete
LeftoversGone
ConverterOnlyDeletes
ConvertsOnlyOnRequest
RootsStick
ChildrenStick
RequestsAnswered
@@ -0,0 +1,612 @@
---------------------------- MODULE V1Conversion ----------------------------
(***************************************************************************)
(* Step: v1 sessions convert to v2 when they are opened (D55, D56, D59, *)
(* D60, D61). *)
(* *)
(* Processes of the new build (Procs) each ask to open a root id, or the *)
(* convert-all command asks for it. If v2 holds the id, the request is *)
(* answered at once, after deleting any v1 folder of the family a crash *)
(* left behind that is unchanged since it was copied. If only v1 holds *)
(* it, the process converts the family, the root and the child sessions *)
(* in its registry, before it answers: it takes the v1 lock of every *)
(* member (a held one answers "busy"), and only then clears the family's *)
(* crash leftovers in .tmp and stages a copy of each member (an *)
(* unreadable member answers "unreadable" and drops the staging). A *)
(* child an earlier, crashed conversion already published is reused as *)
(* it is. It publishes the children and then the root, deletes each *)
(* member's v1 folder under its lock while that folder is the version *)
(* its copy holds (D60), and releases the locks. A delete the file *)
(* system refuses leaves the folder and still answers. Nothing converts *)
(* without a request, and the converter writes no v1 file: it only *)
(* deletes a folder whose copy is published. *)
(* *)
(* Older builds keep writing v1 sessions throughout: a writer holds a *)
(* session's lock while it has the session open, and a write in progress *)
(* leaves the files torn. The new build creates fresh v2 sessions and *)
(* appends to any v2 root. Any process can crash at any step: a crash *)
(* drops every lock it held and leaves staging, published children and *)
(* undeleted v1 folders behind. *)
(* *)
(* Rule R1 (AvoidV1Ids): a fresh v2 session never takes the id of a live *)
(* v1 session. Assumption A1: an older build never creates a v1 session *)
(* under an id v2 holds (ids are 72 random bits). *)
(* *)
(* Converted once per id (D55): a v1 folder an older build changed after *)
(* its copy was made is never converted again or deleted, so v1ver may *)
(* run ahead of base. *)
(* *)
(* Design: system-design.md D53 to D57 and D59 to D61. *)
(***************************************************************************)
EXTENDS Naturals, FiniteSets
CONSTANTS
Procs, \* processes of the new build
RootId, \* a v1 root session
ChildId, \* a child session in that root's registry
NewIds, \* ids only fresh v2 sessions can take
None,
Torn, \* the copy of a member read while a writer was mid-write
MaxV, \* versions a v1 session reaches
MaxLocks, \* times a writer opens one session
MaxExtra, \* v2 turns appended to one session
MaxCrash, \* crashes and refused deletes in one behavior
MaxReq, \* open and convert-all requests in one behavior
AvoidV1Ids \* rule R1
Ids == {RootId, ChildId}
Parent == [i \in Ids |-> IF i = ChildId THEN RootId ELSE None]
ASSUME RootId # ChildId /\ Ids \cap NewIds = {} /\ Procs \cap {"free", "writer"} = {}
AllIds == Ids \cup NewIds
Roots == {i \in Ids : Parent[i] = None}
Children(r) == {c \in Ids : Parent[c] = r}
Fam(r) == {r} \cup Children(r)
Kinds == {"none", "open", "all"}
Answers == {"none", "opened", "converted", "busy", "unreadable", "notfound"}
Status == {"none", "import", "fresh"}
VARIABLES
v1ver, \* [Ids -> 0..MaxV]: 0 absent, else the version of its files
v1lock, \* [Ids -> {"free", "writer"} \cup Procs]
writing, \* [Ids -> BOOLEAN]: a writer is mid-write; the files are torn
locks, \* [Ids -> 0..MaxLocks]: times a writer took the lock
bad, \* [Ids -> BOOLEAN]: the member's v1 files cannot be read
pub, \* [AllIds -> Status]: the id in v2
base, \* [AllIds -> 0..MaxV \cup {Torn}]: the version a conversion
\* copied, which the v2 copy records as its fingerprint
staged, \* [Ids -> BOOLEAN]: a copy in .tmp
stagedBase, \* [Ids -> 0..MaxV \cup {Torn}]
extra, \* [AllIds -> 0..MaxExtra]: v2 turns appended since publish
conv, \* [Procs -> the converter: pc, the family root, its locked
\* members, and those it copied in this run]
req, \* [Procs -> the request being answered: kind and root]
out, \* [Procs -> the answer to the latest request: ans and root]
refused, \* [Procs -> SUBSET Ids]: deletes refused in the current request
reqs, \* requests made
crashes,
famOf, \* [Roots -> SUBSET Ids]: the members a root's conversion covered
mark1, \* [Ids -> BOOLEAN]: v1's usage marker, ~/.fx/usage-recovery/{id}
mark2, \* [Ids -> BOOLEAN]: v2's usage marker, ~/.fx/usage-recovery-v2/{id}
owed \* [Ids -> BOOLEAN]: ghost: the member has usage still to settle
vars == <<v1ver, v1lock, writing, locks, bad, pub, base, staged, stagedBase,
extra, conv, req, out, refused, reqs, crashes, famOf, mark1, mark2, owed>>
marks == <<mark1, mark2, owed>>
Idle == [pc |-> "idle", root |-> None, members |-> {}, copied |-> {}]
NoReq == [kind |-> "none", root |-> None]
TypeOK ==
/\ v1ver \in [Ids -> 0..MaxV]
/\ v1lock \in [Ids -> {"free", "writer"} \cup Procs]
/\ writing \in [Ids -> BOOLEAN]
/\ locks \in [Ids -> 0..MaxLocks]
/\ bad \in [Ids -> BOOLEAN]
/\ pub \in [AllIds -> Status]
/\ base \in [AllIds -> (0..MaxV) \cup {Torn}]
/\ staged \in [Ids -> BOOLEAN]
/\ stagedBase \in [Ids -> (0..MaxV) \cup {Torn}]
/\ extra \in [AllIds -> 0..MaxExtra]
/\ \A p \in Procs :
/\ conv[p].pc \in {"idle", "locked", "staged", "published"}
/\ conv[p].root \in Roots \cup {None}
/\ conv[p].members \subseteq Ids /\ conv[p].copied \subseteq conv[p].members
/\ req[p].kind \in Kinds /\ req[p].root \in Roots \cup {None}
/\ out[p].ans \in Answers /\ out[p].root \in Roots \cup {None}
/\ refused[p] \subseteq Ids
/\ reqs \in 0..MaxReq
/\ crashes \in 0..MaxCrash
/\ famOf \in [Roots -> SUBSET Ids]
/\ mark1 \in [Ids -> BOOLEAN] /\ mark2 \in [Ids -> BOOLEAN] /\ owed \in [Ids -> BOOLEAN]
Live(i) == pub[i] \in {"import", "fresh"}
\* A published child whose root is not published: a crash's leftover, which
\* the next conversion of its family reuses as it is.
Orphan(i) == Parent[i] # None /\ pub[i] = "import" /\ pub[Parent[i]] = "none"
Dirty == \E i \in Ids : staged[i]
\* A v1 folder a crash or a refused delete left behind, unchanged since its
\* copy (damage changes its fingerprint too), free, and not refused in p's
\* current request: p's open deletes it.
Leftover(p, i) ==
/\ pub[i] = "import" /\ v1ver[i] > 0 /\ v1ver[i] = base[i] /\ ~bad[i]
/\ v1lock[i] = "free" /\ i \notin refused[p]
\* What the picker and `fx sessions` list: v2 roots, and v1 roots v2 does not
\* hold; children are internal.
Listed ==
{r \in Roots : Live(r) \/ (pub[r] = "none" /\ v1ver[r] > 0)}
\cup {n \in NewIds : Live(n)}
Init ==
/\ v1ver = [i \in Ids |-> 0]
/\ v1lock = [i \in Ids |-> "free"]
/\ writing = [i \in Ids |-> FALSE]
/\ locks = [i \in Ids |-> 0]
/\ bad = [i \in Ids |-> FALSE]
/\ pub = [i \in AllIds |-> "none"]
/\ base = [i \in AllIds |-> 0]
/\ staged = [i \in Ids |-> FALSE]
/\ stagedBase = [i \in Ids |-> 0]
/\ extra = [i \in AllIds |-> 0]
/\ conv = [p \in Procs |-> Idle]
/\ req = [p \in Procs |-> NoReq]
/\ out = [p \in Procs |-> [ans |-> "none", root |-> None]]
/\ refused = [p \in Procs |-> {}]
/\ reqs = 0
/\ crashes = 0
/\ famOf = [r \in Roots |-> {}]
/\ mark1 = [i \in Ids |-> FALSE]
/\ mark2 = [i \in Ids |-> FALSE]
/\ owed = [i \in Ids |-> FALSE]
(* -- v1 writers: older builds *)
WCreate(i) ==
/\ v1ver[i] = 0
/\ Parent[i] # None => v1ver[Parent[i]] > 0
/\ pub[i] = "none" \* A1
/\ v1ver' = [v1ver EXCEPT ![i] = 1]
/\ UNCHANGED <<v1lock, writing, locks, bad, pub, base, staged, stagedBase,
extra, conv, req, out, refused, reqs, crashes, famOf>>
/\ UNCHANGED marks
WLock(i) ==
/\ v1ver[i] > 0
/\ v1lock[i] = "free"
/\ locks[i] < MaxLocks
/\ v1lock' = [v1lock EXCEPT ![i] = "writer"]
/\ locks' = [locks EXCEPT ![i] = @ + 1]
/\ UNCHANGED <<v1ver, writing, bad, pub, base, staged, stagedBase, extra,
conv, req, out, refused, reqs, crashes, famOf>>
/\ UNCHANGED marks
WBegin(i) ==
/\ v1lock[i] = "writer"
/\ ~writing[i]
/\ v1ver[i] < MaxV
/\ writing' = [writing EXCEPT ![i] = TRUE]
/\ UNCHANGED <<v1ver, v1lock, locks, bad, pub, base, staged, stagedBase,
extra, conv, req, out, refused, reqs, crashes, famOf>>
/\ UNCHANGED marks
WEnd(i) ==
/\ writing[i]
/\ writing' = [writing EXCEPT ![i] = FALSE]
/\ v1ver' = [v1ver EXCEPT ![i] = @ + 1]
/\ UNCHANGED <<v1lock, locks, bad, pub, base, staged, stagedBase, extra,
conv, req, out, refused, reqs, crashes, famOf>>
/\ UNCHANGED marks
WUnlock(i) ==
/\ v1lock[i] = "writer"
/\ ~writing[i]
/\ v1lock' = [v1lock EXCEPT ![i] = "free"]
/\ UNCHANGED <<v1ver, writing, locks, bad, pub, base, staged, stagedBase,
extra, conv, req, out, refused, reqs, crashes, famOf>>
/\ UNCHANGED marks
\* A writer dies: the kernel drops its lock, and v1's own open repairs a torn
\* write back to the last whole version.
WCrash(i) ==
/\ v1lock[i] = "writer"
/\ crashes < MaxCrash
/\ v1lock' = [v1lock EXCEPT ![i] = "free"]
/\ writing' = [writing EXCEPT ![i] = FALSE]
/\ crashes' = crashes + 1
/\ UNCHANGED <<v1ver, locks, bad, pub, base, staged, stagedBase, extra,
conv, req, out, refused, reqs, famOf>>
/\ UNCHANGED marks
\* An older build ends a turn whose usage is not settled yet: it keeps v1's
\* usage marker, under its lock.
WMark(i) ==
/\ v1lock[i] = "writer"
/\ ~mark1[i]
/\ mark1' = [mark1 EXCEPT ![i] = TRUE]
/\ owed' = [owed EXCEPT ![i] = TRUE]
/\ UNCHANGED <<v1ver, v1lock, writing, locks, bad, pub, base, staged, stagedBase,
extra, conv, req, out, refused, reqs, crashes, famOf, mark2>>
\* A member's v1 files become unreadable: damage from outside fx.
Damage(i) ==
/\ v1ver[i] > 0
/\ ~bad[i]
/\ bad' = [bad EXCEPT ![i] = TRUE]
/\ UNCHANGED <<v1ver, v1lock, writing, locks, pub, base, staged,
stagedBase, extra, conv, req, out, refused, reqs, crashes, famOf>>
/\ UNCHANGED marks
(* -- the new build's own v2 sessions *)
FCreate(i) ==
/\ i \in NewIds \/ (i \in Roots /\ (AvoidV1Ids => v1ver[i] = 0))
/\ pub[i] = "none"
/\ i \in Ids => ~staged[i]
/\ pub' = [pub EXCEPT ![i] = "fresh"]
/\ UNCHANGED <<v1ver, v1lock, writing, locks, bad, base, staged,
stagedBase, extra, conv, req, out, refused, reqs, crashes, famOf>>
/\ UNCHANGED marks
FAppend(i) ==
/\ Live(i)
/\ i \in NewIds \/ i \in Roots
/\ extra[i] < MaxExtra
/\ extra' = [extra EXCEPT ![i] = @ + 1]
/\ UNCHANGED <<v1ver, v1lock, writing, locks, bad, pub, base, staged,
stagedBase, conv, req, out, refused, reqs, crashes, famOf>>
/\ UNCHANGED marks
(* -- requests and the converter, in each requesting process *)
Request(p, k, r) ==
/\ k \in {"open", "all"}
/\ req[p] = NoReq
/\ conv[p] = Idle
/\ reqs < MaxReq
/\ req' = [req EXCEPT ![p] = [kind |-> k, root |-> r]]
/\ out' = [out EXCEPT ![p] = [ans |-> "none", root |-> r]]
/\ refused' = [refused EXCEPT ![p] = {}]
/\ reqs' = reqs + 1
/\ UNCHANGED <<v1ver, v1lock, writing, locks, bad, pub, base, staged,
stagedBase, extra, conv, crashes, famOf>>
/\ UNCHANGED marks
Answer(p, a) ==
/\ req' = [req EXCEPT ![p] = NoReq]
/\ out' = [out EXCEPT ![p] = [ans |-> a, root |-> req[p].root]]
Done(p) == IF req[p].kind = "open" THEN "opened" ELSE "converted"
Waiting(p) == req[p] # NoReq /\ conv[p] = Idle
\* v2 already holds the id: any leftover of its family is cleaned first.
AHeld(p) ==
/\ Waiting(p)
/\ Live(req[p].root)
/\ ~\E i \in Fam(req[p].root) : Leftover(p, i)
/\ Answer(p, Done(p))
/\ UNCHANGED <<v1ver, v1lock, writing, locks, bad, pub, base, staged,
stagedBase, extra, conv, refused, reqs, crashes, famOf>>
/\ UNCHANGED marks
\* A leftover v1 folder, unchanged since its copy, is deleted under its lock.
CClean(p, i) ==
/\ Waiting(p)
/\ Live(req[p].root)
/\ i \in Fam(req[p].root)
/\ Leftover(p, i) /\ ~mark1[i]
/\ v1ver' = [v1ver EXCEPT ![i] = 0]
/\ UNCHANGED <<v1lock, writing, locks, bad, pub, base, staged,
stagedBase, extra, conv, req, out, refused, reqs, crashes, famOf>>
/\ UNCHANGED marks
\* The file system refuses that delete: the folder stays, the open goes on.
CCleanRefused(p, i) ==
/\ Waiting(p)
/\ Live(req[p].root)
/\ i \in Fam(req[p].root)
/\ Leftover(p, i)
/\ crashes < MaxCrash
/\ refused' = [refused EXCEPT ![p] = @ \cup {i}]
/\ crashes' = crashes + 1
/\ UNCHANGED <<v1ver, v1lock, writing, locks, bad, pub, base, staged,
stagedBase, extra, conv, req, out, reqs, famOf>>
/\ UNCHANGED marks
\* Neither store holds it.
AMissing(p) ==
/\ Waiting(p)
/\ pub[req[p].root] = "none" /\ v1ver[req[p].root] = 0
/\ Answer(p, IF req[p].kind = "open" THEN "notfound" ELSE "converted")
/\ UNCHANGED <<v1ver, v1lock, writing, locks, bad, pub, base, staged,
stagedBase, extra, conv, refused, reqs, crashes, famOf>>
/\ UNCHANGED marks
\* Takes every present member's lock, or answers "busy" holding none. The
\* check that v2 does not hold the root is made under the locks: another
\* process may have converted it while this one waited.
CBegin(p) ==
LET r == req[p].root
present == {i \in Fam(r) : v1ver[i] > 0} IN
/\ Waiting(p)
/\ pub[r] = "none" /\ v1ver[r] > 0
/\ IF \A i \in present : v1lock[i] = "free"
THEN /\ v1lock' = [i \in Ids |-> IF i \in present THEN p ELSE v1lock[i]]
/\ conv' = [conv EXCEPT ![p] = [pc |-> "locked", root |-> r, members |-> present, copied |-> {}]]
/\ UNCHANGED <<req, out>>
ELSE /\ Answer(p, "busy")
/\ UNCHANGED <<v1lock, conv>>
/\ UNCHANGED <<v1ver, writing, locks, bad, pub, base, staged,
stagedBase, extra, refused, reqs, crashes, famOf>>
/\ UNCHANGED marks
\* Under the locks: the family's staging leftovers go, and each member not
\* already published gets a fresh copy in .tmp; an unreadable member drops
\* the staging and answers "unreadable".
CStage(p) ==
LET m == conv[p].members
copy == {i \in m : pub[i] = "none"} IN
/\ conv[p].pc = "locked"
/\ IF \E i \in copy : bad[i]
THEN /\ v1lock' = [i \in Ids |-> IF i \in m THEN "free" ELSE v1lock[i]]
/\ staged' = [i \in Ids |-> IF i \in Fam(conv[p].root) THEN FALSE ELSE staged[i]]
/\ conv' = [conv EXCEPT ![p] = Idle]
/\ Answer(p, "unreadable")
/\ UNCHANGED stagedBase
ELSE /\ staged' = [i \in Ids |-> IF i \in Fam(conv[p].root) THEN i \in copy ELSE staged[i]]
/\ stagedBase' = [i \in Ids |->
IF i \in copy
THEN (IF writing[i] THEN Torn ELSE v1ver[i])
ELSE stagedBase[i]]
/\ conv' = [conv EXCEPT ![p].pc = "staged", ![p].copied = copy]
/\ UNCHANGED <<v1lock, req, out>>
/\ UNCHANGED <<v1ver, writing, locks, bad, pub, base, extra, refused,
reqs, crashes, famOf>>
/\ UNCHANGED marks
\* Publishing is the manager's atomic rename.
CPublishChild(p, c) ==
/\ conv[p].pc = "staged"
/\ c \in conv[p].members \ {conv[p].root}
/\ staged[c]
/\ staged' = [staged EXCEPT ![c] = FALSE]
/\ pub' = [pub EXCEPT ![c] = "import"]
/\ base' = [base EXCEPT ![c] = stagedBase[c]]
/\ UNCHANGED <<v1ver, v1lock, writing, locks, bad, stagedBase, extra, conv,
req, out, refused, reqs, crashes, famOf>>
/\ UNCHANGED marks
CPublishRoot(p) ==
LET r == conv[p].root IN
/\ conv[p].pc = "staged"
/\ \A c \in conv[p].members \ {r} : ~staged[c]
/\ staged' = [staged EXCEPT ![r] = FALSE]
/\ pub' = [pub EXCEPT ![r] = "import"]
/\ base' = [base EXCEPT ![r] = stagedBase[r]]
/\ conv' = [conv EXCEPT ![p].pc = "published"]
/\ famOf' = [famOf EXCEPT ![r] = conv[p].members]
/\ UNCHANGED <<v1ver, v1lock, writing, locks, bad, stagedBase, extra, req,
out, refused, reqs, crashes>>
/\ UNCHANGED marks
\* A member whose v1 state p may move to v2 and remove: one it copied in
\* this run, or a reused child still at its copy's version and fingerprint.
Unchanged(p, i) ==
/\ i \in conv[p].members /\ v1ver[i] > 0 /\ v1ver[i] = base[i]
/\ i \in conv[p].copied \/ ~bad[i]
\* Its folder goes once its usage marker has moved (D62).
Deletable(p, i) == Unchanged(p, i) /\ ~mark1[i]
\* Where p moves a member's usage marker: after it published that member's
\* family, or for a leftover of a family v2 already holds.
MarkScope(p, i) ==
\/ conv[p].pc = "published" /\ Unchanged(p, i)
\/ Waiting(p) /\ Live(req[p].root) /\ i \in Fam(req[p].root) /\ Leftover(p, i)
\* The v2 marker is written first, then v1's is removed: usage to settle is
\* always behind one of them, and settling both is idempotent.
MoveMark(p, i) ==
/\ MarkScope(p, i)
/\ mark1[i] /\ ~mark2[i]
/\ mark2' = [mark2 EXCEPT ![i] = TRUE]
/\ UNCHANGED <<v1ver, v1lock, writing, locks, bad, pub, base, staged, stagedBase,
extra, conv, req, out, refused, reqs, crashes, famOf, mark1, owed>>
DropMark(p, i) ==
/\ MarkScope(p, i)
/\ mark1[i] /\ mark2[i]
/\ mark1' = [mark1 EXCEPT ![i] = FALSE]
/\ UNCHANGED <<v1ver, v1lock, writing, locks, bad, pub, base, staged, stagedBase,
extra, conv, req, out, refused, reqs, crashes, famOf, mark2, owed>>
\* D60: with the copy durable, each member's v1 folder goes, still locked.
CDelete(p, i) ==
/\ conv[p].pc = "published"
/\ Deletable(p, i)
/\ i \notin refused[p]
/\ v1ver' = [v1ver EXCEPT ![i] = 0]
/\ UNCHANGED <<v1lock, writing, locks, bad, pub, base, staged, stagedBase,
extra, conv, req, out, refused, reqs, crashes, famOf>>
/\ UNCHANGED marks
\* The file system refuses that delete, or the marker move: the folder
\* stays as a leftover.
CDeleteRefused(p, i) ==
/\ conv[p].pc = "published"
/\ Unchanged(p, i)
/\ i \notin refused[p]
/\ crashes < MaxCrash
/\ refused' = [refused EXCEPT ![p] = @ \cup {i}]
/\ crashes' = crashes + 1
/\ UNCHANGED <<v1ver, v1lock, writing, locks, bad, pub, base, staged,
stagedBase, extra, conv, req, out, reqs, famOf>>
/\ UNCHANGED marks
\* Every member's folder is gone, refused, or not deletable (a reused child
\* an older build changed or that was damaged): the locks go, it answers.
CFinish(p) ==
/\ conv[p].pc = "published"
/\ \A i \in conv[p].members : ~Unchanged(p, i) \/ i \in refused[p]
/\ v1lock' = [i \in Ids |-> IF i \in conv[p].members THEN "free" ELSE v1lock[i]]
/\ conv' = [conv EXCEPT ![p] = Idle]
/\ Answer(p, Done(p))
/\ UNCHANGED <<v1ver, writing, locks, bad, pub, base, staged, stagedBase,
extra, refused, reqs, crashes, famOf>>
/\ UNCHANGED marks
\* The process dies: no answer, and its locks go with it.
CCrash(p) ==
/\ req[p] # NoReq
/\ crashes < MaxCrash
/\ v1lock' = [i \in Ids |-> IF v1lock[i] = p THEN "free" ELSE v1lock[i]]
/\ conv' = [conv EXCEPT ![p] = Idle]
/\ req' = [req EXCEPT ![p] = NoReq]
/\ out' = [out EXCEPT ![p] = [ans |-> "none", root |-> req[p].root]]
/\ crashes' = crashes + 1
/\ UNCHANGED <<v1ver, writing, locks, bad, pub, base, staged,
stagedBase, extra, refused, reqs, famOf>>
/\ UNCHANGED marks
\* A converter's steps, without its faults.
ConvWork(p) ==
\/ AHeld(p) \/ AMissing(p) \/ CBegin(p) \/ CStage(p) \/ CPublishRoot(p) \/ CFinish(p)
\/ \E c \in Ids : CPublishChild(p, c) \/ CDelete(p, c) \/ CClean(p, c) \/ MoveMark(p, c) \/ DropMark(p, c)
\* Faults: a refused delete.
ConvRefused(p) == \E c \in Ids : CDeleteRefused(p, c) \/ CCleanRefused(p, c)
ConvStep(p) == ConvWork(p) \/ ConvRefused(p) \/ CCrash(p)
WriterStep(i) == WCreate(i) \/ WLock(i) \/ WBegin(i) \/ WEnd(i) \/ WUnlock(i) \/ WCrash(i) \/ WMark(i)
Next ==
\/ \E p \in Procs :
\/ ConvStep(p)
\/ \E k \in {"open", "all"}, r \in Roots : Request(p, k, r)
\/ \E i \in Ids : WriterStep(i) \/ Damage(i)
\/ \E i \in AllIds : FCreate(i) \/ FAppend(i)
Spec == Init /\ [][Next]_vars
\* Each converter keeps going on a request it holds, and writers release
\* their locks; requests, crashes, writes and opens are bounded.
FairSpec ==
/\ Spec
/\ \A p \in Procs :
/\ WF_vars(AHeld(p)) /\ WF_vars(AMissing(p)) /\ WF_vars(CBegin(p))
/\ WF_vars(CStage(p)) /\ WF_vars(CPublishRoot(p)) /\ WF_vars(CFinish(p))
/\ \A c \in Ids : WF_vars(CPublishChild(p, c)) /\ WF_vars(CDelete(p, c)) /\ WF_vars(CClean(p, c))
/\ \A c \in Ids : WF_vars(MoveMark(p, c)) /\ WF_vars(DropMark(p, c))
/\ \A i \in Ids : WF_vars(WUnlock(i)) /\ WF_vars(WEnd(i))
(* -- properties *)
\* A converter holds every member's lock from staging to the delete, so the
\* copy it publishes is the version v1 has, never a torn one, and the
\* folder it deletes is that version.
LockHeld ==
\A p \in Procs :
conv[p].pc \in {"locked", "staged", "published"} =>
\A i \in conv[p].members : v1lock[i] = p
CopyIsCurrent ==
\A p \in Procs :
conv[p].pc = "staged" =>
\A i \in conv[p].members : staged[i] => stagedBase[i] = v1ver[i]
NoTornImport == \A i \in Ids : pub[i] = "import" => base[i] # Torn
\* A converter's locks never outlive its conversion.
NoStaleConvLock == \A p \in Procs : conv[p].pc = "idle" => \A i \in Ids : v1lock[i] # p
\* A fresh v2 session never takes a live v1 root's id, which would hide that
\* v1 session for good.
NoShadow == \A r \in Roots : v1ver[r] > 0 => pub[r] # "fresh"
\* A published root's conversion covered every member present when it
\* began, and each of those children is published.
ChildrenLanded ==
\A r \in Roots : pub[r] = "import" => \A c \in famOf[r] \ {r} : pub[c] = "import"
\* A child whose root never landed still has its v1 folder: no delete
\* happens before the whole family is published.
OrphansKeepV1 == \A i \in Ids : Orphan(i) => v1ver[i] > 0
\* Usage still to settle always has a marker, v1's or v2's (D62).
UsageKept == \A i \in Ids : owed[i] => (mark1[i] \/ mark2[i])
\* v1's marker never outlives its folder: recovery would then read a
\* session that is gone.
NoDanglingMark == \A i \in Ids : mark1[i] => v1ver[i] > 0
\* The step at which p answers; each answer is judged when it is given.
AnswerStep(p) == req[p] # NoReq /\ req'[p] = NoReq /\ out'[p].ans # "none"
\* "Opened" or "converted" means v2 holds the root (its children by
\* ChildrenLanded), or there is nothing to convert.
AnsweredWhole ==
[][\A p \in Procs :
(AnswerStep(p) /\ out'[p].ans \in {"opened", "converted"}) =>
(pub'[req[p].root] \in {"import", "fresh"} \/ v1ver'[req[p].root] = 0)]_vars
\* A live v1 session is never reported missing.
NoFalseNotFound ==
[][\A p \in Procs :
(AnswerStep(p) /\ out'[p].ans = "notfound") =>
(pub'[req[p].root] = "none" /\ v1ver'[req[p].root] = 0)]_vars
\* A refused request converts nothing: "busy" changes no staging and takes
\* no lock; "unreadable" leaves no staging of the family and no lock.
NothingHalfConverted ==
[][\A p \in Procs :
/\ (AnswerStep(p) /\ out'[p].ans = "busy") =>
(staged' = staged /\ pub' = pub /\ \A i \in Ids : v1lock'[i] # p)
/\ (AnswerStep(p) /\ out'[p].ans = "unreadable") =>
(pub' = pub /\ \A i \in Fam(req[p].root) : ~staged'[i] /\ v1lock'[i] # p)]_vars
\* D60: a v1 folder goes only once its v2 copy is published and is that
\* folder's version, under its converter's lock or free; no writer deletes.
NoLossOnDelete ==
[][\A i \in Ids :
(v1ver[i] > 0 /\ v1ver'[i] = 0) =>
/\ pub[i] = "import" /\ pub'[i] = "import"
/\ base[i] = v1ver[i] /\ ~writing[i]
/\ v1lock[i] # "writer"]_vars
\* An answered open or conversion leaves no v1 folder of the family that
\* equals its copy, except one held by a writer or by another converter
\* (which deletes it as it finishes), damaged since, or whose delete was
\* refused.
LeftoversGone ==
[][\A p \in Procs :
(AnswerStep(p) /\ out'[p].ans \in {"opened", "converted"}) =>
\A i \in Fam(req[p].root) :
(pub'[i] = "import" /\ v1ver'[i] > 0 /\ v1ver'[i] = base'[i])
=> (v1lock'[i] \in {"writer"} \cup (Procs \ {p}) \/ bad'[i] \/ i \in refused'[p])]_vars
\* The converter writes no v1 file: it only deletes v1 folders.
ConverterOnlyDeletes ==
[][(\E p \in Procs : ConvStep(p)) =>
/\ \A i \in Ids : v1ver'[i] \in {v1ver[i], 0}
/\ UNCHANGED <<writing, bad>>]_vars
\* Nothing converts without a request for that root: no background work.
ConvertsOnlyOnRequest ==
[][\A r \in Roots :
(pub[r] = "none" /\ pub'[r] = "import") => \E p \in Procs : req[p].root = r]_vars
\* A live v2 root is never replaced, and a conversion is never redone, nor
\* is a published child's copy, reused or not: once per id.
RootsStick ==
[][\A i \in Roots \cup NewIds :
Live(i) => (pub'[i] = pub[i] /\ base'[i] = base[i])]_vars
ChildrenStick ==
[][\A c \in Ids :
(Parent[c] # None /\ pub[c] = "import") => (pub'[c] = "import" /\ base'[c] = base[c])]_vars
\* Every request is answered, unless its process dies.
RequestsAnswered == \A p \in Procs : (req[p] # NoReq) ~> (req[p] = NoReq)
=============================================================================
+13
View File
@@ -0,0 +1,13 @@
SPECIFICATION Spec
CONSTANT Sessions = {a, b, c}
INVARIANTS
TypeOK
OneBackend
ParentOutlivesChildren
UsageNeverSilent
ContinueLinksInterrupted
SwitchIsTwoSessions
PROPERTIES
ForkIsNewSession
ForkLeavesSource
ForkBetweenTurns
+305
View File
@@ -0,0 +1,305 @@
------------------------------ MODULE Wiring ------------------------------
(***************************************************************************)
(* Step: fx wiring, through the adapter. *)
(* *)
(* One fx process at a time. It picks its backend once at start *)
(* (`--sessions-v2` or FX_SESSIONS_V2, D26) and keeps it until it exits. *)
(* The host has one session slot: it opens a session through the adapter, *)
(* runs turns, may switch to another session, and closes it at exit. To *)
(* switch it may open the next session before closing the current one *)
(* (D41), which fills a second slot, `next`, until the current closes. A *)
(* turn may spawn a child thread, whose finish is appended through the *)
(* parent's Session. Usage is charged to the profile as it happens and *)
(* reaches the session log with the next durable batch; the profile's *)
(* usage-recovery marker covers the gap. "Continue" starts a new turn *)
(* linked to an interrupted one (D23). `/fork` (D58) makes a new session *)
(* from the current one's log before a prompt and switches to it. The *)
(* process can crash anywhere. *)
(* *)
(* The manager's own rules are modeled elsewhere (TurnLifecycle.tla, *)
(* Subagents.tla, WriterLock.tla); here a reopen simply ends a turn a *)
(* crash left open as interrupted (D1). *)
(* *)
(* Design: system-design.md "fx mapping" and D20 to D27. *)
(***************************************************************************)
EXTENDS Naturals, FiniteSets
CONSTANT Sessions
Backends == {"v1", "v2"}
Ends == {"none", "commit", "interrupt"}
VARIABLES
up, \* the fx process is running
backend, \* the backend this process chose at start
wrote, \* backends this process has written since it started
cur, \* the session open in the host slot, or "none"
turnOpen, \* [Sessions -> BOOLEAN]: the log has an open turn
lastEnd, \* [Sessions -> Ends]: how the newest ended turn ended
linked, \* [Sessions -> Ends]: how the turn a continuation continues ended
child, \* "none" or "running": the one child thread
childOf, \* the session whose turn spawned the child, or "none"
usageSplit, \* [Sessions -> BOOLEAN]: the profile and the log may disagree on usage
marker, \* [Sessions -> BOOLEAN]: the profile's usage-recovery marker exists
next, \* the session opened to switch to, or "none" (D41)
made, \* [Sessions -> BOOLEAN]: the session exists on disk
forkOf \* [Sessions -> Sessions \cup {"none"}]: the session a fork came from
vars == <<up, backend, wrote, cur, turnOpen, lastEnd, linked, child, childOf, usageSplit, marker, next, made, forkOf>>
\* Every component uses the backend the process chose (D26).
ChildBackend == backend
UsageBackend == backend
TypeOK ==
/\ up \in BOOLEAN
/\ backend \in Backends
/\ wrote \subseteq Backends
/\ cur \in Sessions \cup {"none"}
/\ turnOpen \in [Sessions -> BOOLEAN]
/\ lastEnd \in [Sessions -> Ends]
/\ linked \in [Sessions -> Ends]
/\ child \in {"none", "running"}
/\ childOf \in Sessions \cup {"none"}
/\ usageSplit \in [Sessions -> BOOLEAN]
/\ marker \in [Sessions -> BOOLEAN]
/\ next \in Sessions \cup {"none"}
/\ made \in [Sessions -> BOOLEAN]
/\ forkOf \in [Sessions -> Sessions \cup {"none"}]
Init ==
/\ up = FALSE
/\ backend = "v1"
/\ wrote = {}
/\ cur = "none"
/\ turnOpen = [s \in Sessions |-> FALSE]
/\ lastEnd = [s \in Sessions |-> "none"]
/\ linked = [s \in Sessions |-> "none"]
/\ child = "none"
/\ childOf = "none"
/\ usageSplit = [s \in Sessions |-> FALSE]
/\ marker = [s \in Sessions |-> FALSE]
/\ next = "none"
/\ made = [s \in Sessions |-> FALSE]
/\ forkOf = [s \in Sessions |-> "none"]
Active == up /\ cur # "none"
\* fx starts and picks its backend, once.
Start(b) ==
/\ ~up
/\ up' = TRUE
/\ backend' = b
/\ wrote' = {}
/\ UNCHANGED <<cur, turnOpen, lastEnd, linked, child, childOf, usageSplit, marker, next, made, forkOf>>
\* Startup usage recovery: a marker left by a crash is reconciled, then removed.
RecoverUsage(s) ==
/\ up
/\ marker[s]
/\ usageSplit' = [usageSplit EXCEPT ![s] = FALSE]
/\ marker' = [marker EXCEPT ![s] = FALSE]
/\ UNCHANGED <<up, backend, wrote, cur, turnOpen, lastEnd, linked, child, childOf, next, made, forkOf>>
\* The adapter opens a session: into the slot, or, while one is open, as the
\* session to switch to (D41). A turn a crash left open ends as interrupted.
Open(s) ==
/\ up
/\ next = "none"
/\ s # cur
/\ IF cur = "none" THEN cur' = s /\ next' = "none" ELSE cur' = cur /\ next' = s
/\ wrote' = wrote \cup {backend}
/\ turnOpen' = [turnOpen EXCEPT ![s] = FALSE]
/\ lastEnd' = IF turnOpen[s] THEN [lastEnd EXCEPT ![s] = "interrupt"] ELSE lastEnd
/\ made' = [made EXCEPT ![s] = TRUE]
/\ UNCHANGED <<up, backend, linked, child, childOf, usageSplit, marker, forkOf>>
\* New work needs a session and no switch pending.
Idle == Active /\ next = "none"
BeginTurn ==
/\ Idle
/\ ~turnOpen[cur]
/\ turnOpen' = [turnOpen EXCEPT ![cur] = TRUE]
/\ linked' = [linked EXCEPT ![cur] = "none"]
/\ wrote' = wrote \cup {backend}
/\ UNCHANGED <<up, backend, cur, lastEnd, child, childOf, usageSplit, marker, next, made, forkOf>>
\* "Continue" (D23): a new turn whose first item links to the newest ended
\* turn, which must have been interrupted.
Continue ==
/\ Idle
/\ ~turnOpen[cur]
/\ lastEnd[cur] = "interrupt"
/\ turnOpen' = [turnOpen EXCEPT ![cur] = TRUE]
/\ linked' = [linked EXCEPT ![cur] = lastEnd[cur]]
/\ wrote' = wrote \cup {backend}
/\ UNCHANGED <<up, backend, cur, lastEnd, child, childOf, usageSplit, marker, next, made, forkOf>>
\* Before a model request is charged, the marker is written.
MarkUsage ==
/\ Active
/\ turnOpen[cur]
/\ ~marker[cur]
/\ marker' = [marker EXCEPT ![cur] = TRUE]
/\ UNCHANGED <<up, backend, wrote, cur, turnOpen, lastEnd, linked, child, childOf, usageSplit, next, made, forkOf>>
\* The request is charged to the profile; the log has not caught up yet.
Charge ==
/\ Active
/\ turnOpen[cur]
/\ marker[cur]
/\ usageSplit' = [usageSplit EXCEPT ![cur] = TRUE]
/\ UNCHANGED <<up, backend, wrote, cur, turnOpen, lastEnd, linked, child, childOf, marker, next, made, forkOf>>
\* A usage checkpoint mid-turn: `set usage` is a durability point, so the
\* log catches up with the profile once the append returns.
PersistUsage ==
/\ Active
/\ usageSplit[cur]
/\ usageSplit' = [usageSplit EXCEPT ![cur] = FALSE]
/\ wrote' = wrote \cup {UsageBackend}
/\ UNCHANGED <<up, backend, cur, turnOpen, lastEnd, linked, child, childOf, marker, next, made, forkOf>>
\* The turn ends, also while a switch is pending (a cancelled turn); the
\* usage so far is written in the same durable batch.
EndTurn(e) ==
/\ Active
/\ turnOpen[cur]
/\ turnOpen' = [turnOpen EXCEPT ![cur] = FALSE]
/\ lastEnd' = [lastEnd EXCEPT ![cur] = e]
/\ usageSplit' = [usageSplit EXCEPT ![cur] = FALSE]
/\ wrote' = wrote \cup {UsageBackend}
/\ UNCHANGED <<up, backend, cur, linked, child, childOf, marker, next, made, forkOf>>
\* Once a durable batch carried the usage, the marker goes.
ClearMarker(s) ==
/\ up
/\ marker[s]
/\ ~usageSplit[s]
/\ marker' = [marker EXCEPT ![s] = FALSE]
/\ UNCHANGED <<up, backend, wrote, cur, turnOpen, lastEnd, linked, child, childOf, usageSplit, next, made, forkOf>>
\* A turn spawns a child thread (child_spawned through the parent).
Spawn ==
/\ Idle
/\ turnOpen[cur]
/\ child = "none"
/\ child' = "running"
/\ childOf' = cur
/\ wrote' = wrote \cup {backend, ChildBackend}
/\ UNCHANGED <<up, backend, cur, turnOpen, lastEnd, linked, usageSplit, marker, next, made, forkOf>>
\* The child's thread appends child_finished through the parent's Session,
\* or the parent cancels it; either way through an open parent.
ChildDone ==
/\ up
/\ child = "running"
/\ cur = childOf
/\ child' = "none"
/\ childOf' = "none"
/\ wrote' = wrote \cup {backend}
/\ UNCHANGED <<up, backend, cur, turnOpen, lastEnd, linked, usageSplit, marker, next, made, forkOf>>
\* The host closes a session, to switch or to exit: children are finished
\* and joined first, then close interrupts an open turn and carries usage.
\* Closing the current session hands the slot to the one switched to;
\* closing that one instead abandons the switch (D41).
Close(s) ==
/\ up
/\ s \in {cur, next} \ {"none"}
/\ child = "none"
/\ cur' = IF s = cur THEN next ELSE cur
/\ next' = "none"
/\ turnOpen' = [turnOpen EXCEPT ![s] = FALSE]
/\ lastEnd' = IF turnOpen[s] THEN [lastEnd EXCEPT ![s] = "interrupt"] ELSE lastEnd
/\ usageSplit' = [usageSplit EXCEPT ![s] = FALSE]
/\ wrote' = wrote \cup {backend}
/\ UNCHANGED <<up, backend, linked, child, childOf, marker, made, forkOf>>
\* /fork (D58): a new session holding the current one's log before a chosen
\* prompt becomes the session to switch to, and Close then hands it the
\* slot. Only on v2, between turns and with no child running; the source
\* is never written. The fork's newest ended turn is whichever the source
\* had before that prompt.
Fork(f) ==
/\ Idle
/\ backend = "v2"
/\ child = "none"
/\ f \notin {cur, next}
/\ ~made[f]
/\ ~turnOpen[cur]
/\ next' = f
/\ made' = [made EXCEPT ![f] = TRUE]
/\ forkOf' = [forkOf EXCEPT ![f] = cur]
/\ \E e \in Ends : lastEnd' = [lastEnd EXCEPT ![f] = e]
/\ wrote' = wrote \cup {backend}
/\ UNCHANGED <<up, backend, cur, turnOpen, linked, child, childOf, usageSplit, marker>>
Exit ==
/\ up
/\ cur = "none"
/\ next = "none"
/\ child = "none"
/\ up' = FALSE
/\ UNCHANGED <<backend, wrote, cur, turnOpen, lastEnd, linked, child, childOf, usageSplit, marker, next, made, forkOf>>
\* The process dies anywhere: memory and threads are gone; the logs, the
\* profile ledger and the marker stay.
Crash ==
/\ up
/\ up' = FALSE
/\ cur' = "none"
/\ next' = "none"
/\ child' = "none"
/\ childOf' = "none"
/\ UNCHANGED <<backend, wrote, turnOpen, lastEnd, linked, usageSplit, marker, made, forkOf>>
Next ==
\/ \E b \in Backends : Start(b)
\/ \E s \in Sessions : RecoverUsage(s) \/ Open(s) \/ ClearMarker(s) \/ Close(s)
\/ BeginTurn \/ Continue
\/ MarkUsage \/ Charge \/ PersistUsage
\/ \E e \in {"commit", "interrupt"} : EndTurn(e)
\/ Spawn \/ ChildDone
\/ \E f \in Sessions : Fork(f)
\/ Exit \/ Crash
Spec == Init /\ [][Next]_vars
-----------------------------------------------------------------------------
(* Invariants *)
\* A process writes one backend only: no dual writes.
OneBackend == Cardinality(wrote) <= 1
\* A child thread runs only while its parent's Session is open.
ParentOutlivesChildren == child = "running" => (up /\ cur = childOf)
\* Whenever the profile and the log may disagree on usage, the marker says so.
UsageNeverSilent == \A s \in Sessions : usageSplit[s] => marker[s]
\* A pending switch names a second session, never the current one (D41).
SwitchIsTwoSessions == next # "none" => (cur # "none" /\ cur # next)
\* A continuation always continues an interrupted turn.
ContinueLinksInterrupted == \A s \in Sessions : linked[s] \in {"none", "interrupt"}
(* Properties of /fork (D58) *)
ForkStep(f) == forkOf'[f] # forkOf[f]
\* A fork is a session that never existed: it never replaces one.
ForkIsNewSession == [][\A f \in Sessions : ForkStep(f) => ~made[f]]_vars
\* The fork step leaves the source as it was.
ForkLeavesSource ==
[][\A f \in Sessions : ForkStep(f) =>
/\ turnOpen'[cur] = turnOpen[cur] /\ lastEnd'[cur] = lastEnd[cur]
/\ linked'[cur] = linked[cur] /\ usageSplit'[cur] = usageSplit[cur]]_vars
\* A fork happens on v2, between turns, with no child running.
ForkBetweenTurns ==
[][\A f \in Sessions : ForkStep(f) =>
(backend = "v2" /\ ~turnOpen[cur] /\ child = "none")]_vars
=============================================================================
@@ -0,0 +1,12 @@
SPECIFICATION Spec
CONSTANTS
Procs = {p1, p2}
Threads = {t1, t2}
MaxLines = 4
None = None
INVARIANTS
TypeOK
OneWriter
OnlyHolderOpen
MutexInsideOpenSession
SeqIsPosition
+122
View File
@@ -0,0 +1,122 @@
---------------------------- MODULE WriterLock ----------------------------
(***************************************************************************)
(* Step: L0 flock plus the L2 per-session mutex. *)
(* *)
(* One writing process per session (flock on {id}/lock) and many threads *)
(* per process (the Session mutex inside the manager). The kernel drops *)
(* the flock when a process dies. Hosts never lock anything; Acquire and *)
(* WriteAndRelease happen inside append(). Busy is modeled as Open being *)
(* disabled while another process holds the lock. *)
(* *)
(* Design: system-design.md "Concurrency And Ordering". *)
(***************************************************************************)
EXTENDS Naturals, Sequences
CONSTANTS Procs, Threads, MaxLines, None
VARIABLES
holder, \* process holding the flock, or None
alive, \* [Procs -> BOOLEAN]
opened, \* [Procs -> BOOLEAN]: the process has an open Session
nextSeq, \* [Procs -> Nat]: the Session's in-memory next seq
mutex, \* [Procs -> Threads \cup {None}]: thread inside append()
file \* the log: a sequence of [seq, by]
vars == <<holder, alive, opened, nextSeq, mutex, file>>
TypeOK ==
/\ holder \in Procs \cup {None}
/\ alive \in [Procs -> BOOLEAN]
/\ opened \in [Procs -> BOOLEAN]
/\ nextSeq \in [Procs -> 1..(MaxLines + 1)]
/\ mutex \in [Procs -> Threads \cup {None}]
/\ file \in Seq([seq : 1..MaxLines, by : Procs])
Init ==
/\ holder = None
/\ alive = [p \in Procs |-> TRUE]
/\ opened = [p \in Procs |-> FALSE]
/\ nextSeq = [p \in Procs |-> 1]
/\ mutex = [p \in Procs |-> None]
/\ file = <<>>
\* open(): flock succeeds only when nobody holds it; otherwise Busy.
\* The Session learns the next seq from the file it just locked.
Open(p) ==
/\ alive[p]
/\ ~opened[p]
/\ holder = None
/\ holder' = p
/\ opened' = [opened EXCEPT ![p] = TRUE]
/\ nextSeq' = [nextSeq EXCEPT ![p] = Len(file) + 1]
/\ UNCHANGED <<alive, mutex, file>>
\* A thread calls append(); the manager takes the Session mutex.
Acquire(p, t) ==
/\ alive[p]
/\ opened[p]
/\ mutex[p] = None
/\ mutex' = [mutex EXCEPT ![p] = t]
/\ UNCHANGED <<holder, alive, opened, nextSeq, file>>
\* Under the mutex: assign seq, write the line, release the mutex.
WriteAndRelease(p, t) ==
/\ alive[p]
/\ opened[p]
/\ mutex[p] = t
/\ Len(file) < MaxLines
/\ file' = Append(file, [seq |-> nextSeq[p], by |-> p])
/\ nextSeq' = [nextSeq EXCEPT ![p] = @ + 1]
/\ mutex' = [mutex EXCEPT ![p] = None]
/\ UNCHANGED <<holder, alive, opened>>
\* close(): waits for the mutex, then releases the flock.
Close(p) ==
/\ alive[p]
/\ opened[p]
/\ mutex[p] = None
/\ holder = p
/\ holder' = None
/\ opened' = [opened EXCEPT ![p] = FALSE]
/\ UNCHANGED <<alive, nextSeq, mutex, file>>
\* The process dies; the kernel releases its flock.
Crash(p) ==
/\ alive[p]
/\ alive' = [alive EXCEPT ![p] = FALSE]
/\ opened' = [opened EXCEPT ![p] = FALSE]
/\ mutex' = [mutex EXCEPT ![p] = None]
/\ holder' = IF holder = p THEN None ELSE holder
/\ UNCHANGED <<nextSeq, file>>
Restart(p) ==
/\ ~alive[p]
/\ alive' = [alive EXCEPT ![p] = TRUE]
/\ UNCHANGED <<holder, opened, nextSeq, mutex, file>>
Next ==
\E p \in Procs :
\/ Open(p)
\/ Close(p)
\/ Crash(p)
\/ Restart(p)
\/ \E t \in Threads : Acquire(p, t) \/ WriteAndRelease(p, t)
Spec == Init /\ [][Next]_vars
-----------------------------------------------------------------------------
(* Invariants *)
\* At most one process has the session open for writing.
OneWriter == \A p, q \in Procs : opened[p] /\ opened[q] => p = q
\* Only the flock holder has an open Session.
OnlyHolderOpen == \A p \in Procs : opened[p] => holder = p
\* A thread inside append() always belongs to an open Session.
MutexInsideOpenSession == \A p \in Procs : mutex[p] # None => opened[p]
\* File order is seq order: no duplicates, no gaps, across threads and
\* across processes taking over after a crash.
SeqIsPosition == \A i \in 1..Len(file) : file[i].seq = i
=============================================================================
+58
View File
@@ -0,0 +1,58 @@
#!/bin/sh
# Model-check every session-manager spec with TLC.
#
# Needs Java 11+ and tla2tools.jar:
# brew install openjdk
# mkdir -p ~/.local/share/tla
# curl -fL -o ~/.local/share/tla/tla2tools.jar \
# https://github.com/tlaplus/tlaplus/releases/latest/download/tla2tools.jar
#
# Usage:
# ./check.sh # every spec
# ./check.sh SessionLog Fork # selected specs
# TLA2TOOLS_JAR=/path/tla2tools.jar ./check.sh
set -eu
cd "$(dirname "$0")"
jar="${TLA2TOOLS_JAR:-$HOME/.local/share/tla/tla2tools.jar}"
if [ ! -f "$jar" ]; then
echo "tla2tools.jar not found at $jar (set TLA2TOOLS_JAR)" >&2
exit 2
fi
# Prefer JAVA_HOME, then Homebrew's keg-only openjdk, then java on PATH.
java=""
for candidate in \
"${JAVA_HOME:+$JAVA_HOME/bin/java}" \
/opt/homebrew/opt/openjdk/bin/java \
/usr/local/opt/openjdk/bin/java \
"$(command -v java 2>/dev/null || true)"; do
if [ -n "$candidate" ] && [ -x "$candidate" ] && "$candidate" -version >/dev/null 2>&1; then
java="$candidate"
break
fi
done
if [ -z "$java" ]; then
echo "no working Java runtime (set JAVA_HOME or brew install openjdk)" >&2
exit 2
fi
all="SessionLog TurnLifecycle WriterLock ResumeSnapshot Fork Subagents Catalog Lifecycle Wiring MoveSideFiles V1Conversion"
specs="${*:-$all}"
work="${TMPDIR:-/tmp}/session-manager-tlc"
mkdir -p "$work"
status=0
for spec in $specs; do
echo "== $spec"
# -deadlock: do not report states with no enabled action. Every spec is
# bounded (MaxLen, MaxLines, finite id sets), so terminal states are
# expected at the bound.
if ! "$java" -XX:+UseParallelGC -cp "$jar" tlc2.TLC \
-deadlock -workers auto -cleanup \
-metadir "$work/$spec" \
-config "$spec.cfg" "$spec.tla"; then
status=1
fi
done
exit "$status"
+199
View File
@@ -0,0 +1,199 @@
#!/usr/bin/env python3
"""Check that every spec still catches the bugs it exists to catch.
Each mutant plants one realistic design bug in a temporary copy of a spec
and runs TLC on it. The run passes only if TLC reports the expected
invariant as violated. Run after changing any spec:
./mutants.py
Needs the same Java runtime and tla2tools.jar as check.sh.
"""
import os
import pathlib
import shutil
import subprocess
import sys
import tempfile
HERE = pathlib.Path(__file__).resolve().parent
JAR = os.environ.get("TLA2TOOLS_JAR", os.path.expanduser("~/.local/share/tla/tla2tools.jar"))
# (spec, bug description, text to replace, replacement, invariant that must fail)
MUTANTS = [
("Wiring", "a fork reuses a session that exists",
" /\\ f \\notin {cur, next}\n /\\ ~made[f]\n",
" /\\ f \\notin {cur, next}\n",
"ForkIsNewSession"),
("Wiring", "a fork while a turn is open",
" /\\ ~made[f]\n /\\ ~turnOpen[cur]\n /\\ next' = f",
" /\\ ~made[f]\n /\\ next' = f",
"ForkBetweenTurns"),
("Wiring", "the fork step writes the source",
" /\\ \\E e \\in Ends : lastEnd' = [lastEnd EXCEPT ![f] = e]",
" /\\ \\E e \\in Ends : lastEnd' = [lastEnd EXCEPT ![f] = e, ![cur] = \"interrupt\"]",
"ForkLeavesSource"),
('V1Conversion', 'a converter takes locks another process holds', ' /\\ IF \\A i \\in present : v1lock[i] = "free"\n', ' /\\ IF TRUE\n', 'LockHeld'),
('V1Conversion', 'a fresh v2 session takes a live v1 id', '(AvoidV1Ids => v1ver[i] = 0)', 'TRUE', 'NoShadow'),
('V1Conversion', 'the root lands before its children', ' /\\ \\A c \\in conv[p].members \\ {r} : ~staged[c]\n', '', 'ChildrenLanded'),
('V1Conversion', "a busy answer sweeps another process's staging", ' ELSE /\\ Answer(p, "busy")\n /\\ UNCHANGED <<v1lock, conv>>\n /\\ UNCHANGED <<v1ver, writing, locks, bad, pub, base, staged,\n stagedBase,', ' ELSE /\\ Answer(p, "busy")\n /\\ UNCHANGED <<v1lock, conv>>\n /\\ staged\' = [i \\in Ids |-> FALSE]\n /\\ UNCHANGED <<v1ver, writing, locks, bad, pub, base,\n stagedBase,', 'NothingHalfConverted'),
('V1Conversion', "a crash keeps the converter's locks", ' /\\ v1lock\' = [i \\in Ids |-> IF v1lock[i] = p THEN "free" ELSE v1lock[i]]\n', " /\\ v1lock' = v1lock\n", 'NoStaleConvLock'),
('V1Conversion', 'an open reports a live v1 session missing', ' /\\ pub[req[p].root] = "none" /\\ v1ver[req[p].root] = 0\n /\\ Answer(p, IF', ' /\\ ~Live(req[p].root)\n /\\ Answer(p, IF', 'NoFalseNotFound'),
('V1Conversion', 'a conversion starts without a request', ' LET r == req[p].root\n present == {i \\in Fam(r) : v1ver[i] > 0} IN\n /\\ Waiting(p)\n', ' LET r == RootId\n present == {i \\in Fam(r) : v1ver[i] > 0} IN\n /\\ conv[p] = Idle\n', 'ConvertsOnlyOnRequest'),
('V1Conversion', 'an unreadable member leaves its staging', " /\\ staged' = [i \\in Ids |-> IF i \\in Fam(conv[p].root) THEN FALSE ELSE staged[i]]\n", " /\\ staged' = [i \\in Ids |-> IF i \\in copy THEN TRUE ELSE staged[i]]\n", 'NothingHalfConverted'),
('V1Conversion', 'a busy answer keeps the locks it took', ' ELSE /\\ Answer(p, "busy")\n /\\ UNCHANGED <<v1lock, conv>>', ' ELSE /\\ Answer(p, "busy")\n /\\ v1lock\' = [i \\in Ids |-> IF i \\in present /\\ v1lock[i] = "free" THEN p ELSE v1lock[i]]\n /\\ UNCHANGED conv', 'NoStaleConvLock'),
('V1Conversion', 'a child loses its v1 folder before its root is published', " /\\ base' = [base EXCEPT ![c] = stagedBase[c]]\n /\\ UNCHANGED <<v1ver, v1lock,", " /\\ base' = [base EXCEPT ![c] = stagedBase[c]]\n /\\ v1ver' = [v1ver EXCEPT ![c] = 0]\n /\\ UNCHANGED <<v1lock,", 'OrphansKeepV1'),
('V1Conversion', 'a cleanup deletes a leftover an older build changed', ' /\\ pub[i] = "import" /\\ v1ver[i] > 0 /\\ v1ver[i] = base[i] /\\ ~bad[i]\n', ' /\\ pub[i] = "import" /\\ v1ver[i] > 0 /\\ ~bad[i]\n', 'NoLossOnDelete'),
('V1Conversion', "a cleanup ignores a writer's lock", ' /\\ v1lock[i] = "free" /\\ i \\notin refused[p]', ' /\\ i \\notin refused[p]', 'NoLossOnDelete'),
('V1Conversion', 'a conversion keeps the v1 folders', ' /\\ \\A i \\in conv[p].members : ~Unchanged(p, i) \\/ i \\in refused[p]\n', '', 'LeftoversGone'),
('V1Conversion', 'an open skips the leftover cleanup', ' /\\ ~\\E i \\in Fam(req[p].root) : Leftover(p, i)\n', '', 'LeftoversGone'),
('V1Conversion', 'the locks go before the delete', ' /\\ UNCHANGED <<v1ver, v1lock, writing, locks, bad, stagedBase, extra, req,\n out, refused, reqs, crashes>>', ' /\\ v1lock\' = [i \\in Ids |-> IF i \\in conv[p].members THEN "free" ELSE v1lock[i]]\n /\\ UNCHANGED <<v1ver, writing, locks, bad, stagedBase, extra, req,\n out, refused, reqs, crashes>>', 'LockHeld'),
('V1Conversion', 'the converter writes a v1 file', ' /\\ UNCHANGED <<v1ver, writing, locks, bad, pub, base, extra, refused,\n reqs, crashes, famOf>>', " /\\ bad' = [i \\in Ids |-> IF i \\in conv[p].members THEN TRUE ELSE bad[i]]\n /\\ UNCHANGED <<v1ver, writing, locks, pub, base, extra, refused,\n reqs, crashes, famOf>>", 'ConverterOnlyDeletes'),
('V1Conversion', 'a reused child is deleted after an older build changed it', ' /\\ i \\in conv[p].members /\\ v1ver[i] > 0 /\\ v1ver[i] = base[i]\n /\\ i \\in conv[p].copied', ' /\\ i \\in conv[p].members /\\ v1ver[i] > 0\n /\\ i \\in conv[p].copied', 'NoLossOnDelete'),
('V1Conversion', 'the v1 usage marker goes before the v2 one is written', " /\\ mark1[i] /\\ mark2[i]\n /\\ mark1' = [mark1 EXCEPT ![i] = FALSE]", " /\\ mark1[i]\n /\\ mark1' = [mark1 EXCEPT ![i] = FALSE]", 'UsageKept'),
('V1Conversion', 'a folder goes before its usage marker moved', 'Deletable(p, i) == Unchanged(p, i) /\\ ~mark1[i]', 'Deletable(p, i) == Unchanged(p, i)', 'NoDanglingMark'),
('V1Conversion', 'a leftover is cleaned before its usage marker moved', " /\\ Leftover(p, i) /\\ ~mark1[i]\n /\\ v1ver' = [v1ver EXCEPT ![i] = 0]", " /\\ Leftover(p, i)\n /\\ v1ver' = [v1ver EXCEPT ![i] = 0]", 'NoDanglingMark'),
('V1Conversion', 'a reused child is copied again', ' copy == {i \\in m : pub[i] = "none"} IN', ' copy == m IN', 'ChildrenStick'),
("SessionLog", "fsync acks a half-written line",
'Fsync ==\n /\\ status = "open"',
'Fsync ==\n /\\ status \\in {"open", "writing"}',
"AckedSurvive"),
("SessionLog", "open() keeps a torn tail",
"IF LastDone THEN log ELSE SubSeq(log, 1, Len(log) - 1)", "log",
"OpenIsClean"),
("TurnLifecycle", "reopen skips turn_interrupted",
"log' = IF IsOpen(log) THEN Append(log, \"interrupt\") ELSE log", "log' = log",
"NoStaleTurnAfterReopen"),
("TurnLifecycle", "manager accepts an item outside a turn",
'[] k = "item" -> IsOpen(l)', '[] k = "item" -> TRUE',
"ItemsInsideTurns"),
("WriterLock", "open ignores the flock",
" /\\ holder = None\n /\\ holder' = p", " /\\ holder' = p",
"OneWriter"),
("ResumeSnapshot", "snapshot stores the wrong state",
'Entry("snap", NoVal, Full(log))', 'Entry("snap", NoVal, Empty)',
"ResumeEqualsFullReplay"),
("Fork", "fork forgets blob links",
"links' = [links EXCEPT ![f] = links[src]]", "links' = links",
"RefsExist"),
("Fork", "fork copies a line whose blob was lost",
" /\\ \\A i \\in 2..at : lg[src][i].b # NoBlob => lg[src][i].b \\in links[src]\n", "",
"RefsExist"),
("Fork", "fork allowed inside an open turn",
" /\\ ~IsOpen(SubSeq(lg[src], 1, at))\n", "",
"ForkAtBoundary"),
("Subagents", "repair marks every child lost",
'o |-> IF childLog[c] THEN "interrupted" ELSE "lost"', 'o |-> "lost"',
"LostMeansNoLog"),
("Subagents", "spawn while the child's work is open",
" /\\ ~Unfinished(c)\n /\\ Works(c) < MaxWork",
" /\\ Works(c) < MaxWork",
"OneOpenWork"),
("Wiring", "a subagent opens its session on v1",
"ChildBackend == backend", 'ChildBackend == "v1"',
"OneBackend"),
("Wiring", "close without joining child threads",
" /\\ child = \"none\"\n /\\ cur' = IF s = cur",
" /\\ cur' = IF s = cur",
"ParentOutlivesChildren"),
("Wiring", "switch to the session already open",
" /\\ s # cur\n",
"",
"SwitchIsTwoSessions"),
("Wiring", "charge usage before writing the marker",
" /\\ marker[cur]\n /\\ usageSplit'",
" /\\ usageSplit'",
"UsageNeverSilent"),
("Wiring", "clear the marker before usage is durable",
" /\\ marker[s]\n /\\ ~usageSplit[s]\n",
" /\\ marker[s]\n",
"UsageNeverSilent"),
("Wiring", "continue a committed turn",
" /\\ lastEnd[cur] = \"interrupt\"\n",
"",
"ContinueLinksInterrupted"),
("Subagents", "reopen skips child repair",
" /\\ repairing' = TRUE\n /\\ UNCHANGED <<plog, childLog, thread>>",
" /\\ repairing' = FALSE\n /\\ UNCHANGED <<plog, childLog, thread>>",
"NoIndeterminate"),
("Catalog", "list skips the folder check",
'Listed == {s \\in Indexed : dir[s] = "live"}', "Listed == Indexed",
"ListedAreLive"),
("Fork", "a setting lists a blob the session lacks",
" /\\ b \\in links[s]\n /\\ lg' = [lg EXCEPT ![s] = Append(@, E(\"set\", b))]",
" /\\ lg' = [lg EXCEPT ![s] = Append(@, E(\"set\", b))]",
"RefsExist"),
("MoveSideFiles", "the map is written before every body is a blob",
" /\\ ~mapped\n /\\ blobs = Handles\n /\\ mapped' = TRUE",
" /\\ ~mapped\n /\\ mapped' = TRUE",
"Resolvable"),
("MoveSideFiles", "the side folder goes before the terminal moved",
" /\\ mapped\n /\\ terminal = \"moved\"\n /\\ folder = \"present\"",
" /\\ mapped\n /\\ folder = \"present\"",
"TerminalKept"),
("Catalog", "an index refresh for a session never published",
'Refresh(s) ==\n /\\ dir[s] = "live"', "Refresh(s) ==\n /\\ TRUE",
"IndexNamesPublished"),
("Lifecycle", "rename before fsync of line 1",
'Advance(s, "synced", "renamed")', 'Advance(s, "written", "renamed")',
"VisibleHasLine1"),
("Lifecycle", "ack before fsync of the directory",
'Advance(s, "renamed", "done")', 'Advance(s, "synced", "done")',
"AckedIsVisible"),
]
def find_java():
candidates = []
if os.environ.get("JAVA_HOME"):
candidates.append(os.path.join(os.environ["JAVA_HOME"], "bin", "java"))
candidates += ["/opt/homebrew/opt/openjdk/bin/java", "/usr/local/opt/openjdk/bin/java"]
on_path = shutil.which("java")
if on_path:
candidates.append(on_path)
for candidate in candidates:
if os.access(candidate, os.X_OK):
probe = subprocess.run([candidate, "-version"], capture_output=True)
if probe.returncode == 0:
return candidate
return None
def main():
java = find_java()
if java is None or not os.path.isfile(JAR):
print("needs a Java runtime and tla2tools.jar (see check.sh)", file=sys.stderr)
return 2
survivors = 0
with tempfile.TemporaryDirectory(prefix="session-manager-mutants-") as tmp:
for index, (spec, desc, old, new, expected) in enumerate(MUTANTS):
work = pathlib.Path(tmp) / f"{index:02d}-{spec}"
work.mkdir()
text = (HERE / f"{spec}.tla").read_text()
if old not in text:
print(f"{spec:15s} {desc:42s} STALE: mutation no longer applies")
survivors += 1
continue
(work / f"{spec}.tla").write_text(text.replace(old, new, 1))
# A smaller model, when the full one is too large to search per mutant.
small = HERE / f"{spec}.mutants.cfg"
shutil.copy(small if small.exists() else HERE / f"{spec}.cfg", work / f"{spec}.cfg")
run = subprocess.run(
[java, "-XX:+UseParallelGC", "-cp", JAR, "tlc2.TLC", "-deadlock",
"-workers", "auto", "-cleanup", "-metadir", str(work / "states"),
"-config", f"{spec}.cfg", f"{spec}.tla"],
cwd=work, capture_output=True, text=True, timeout=900,
)
output = run.stdout + run.stderr
if (f"Invariant {expected} is violated" in output
or f"Action property {expected} is violated" in output):
print(f"{spec:15s} {desc:42s} caught by {expected}")
else:
print(f"{spec:15s} {desc:42s} SURVIVED (expected {expected} to fail)")
survivors += 1
print("all mutants caught" if survivors == 0 else f"{survivors} mutant(s) survived")
return 0 if survivors == 0 else 1
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,4 @@
SPECIFICATION TraceSpec
CONSTANT Ids <- TraceIds
CONSTRAINT Progress
INVARIANT TraceIncomplete
@@ -0,0 +1,57 @@
---------------------------- MODULE CatalogTrace ----------------------------
(***************************************************************************)
(* Trace validation for Catalog.tla (index.jsonl, delete, heal, rebuild). *)
(* *)
(* One trace line per action: *)
(* event Publish | IndexPut | Trash | IndexDel | Purge | Heal (with *)
(* s) | Crash | Rebuild *)
(* s "s1" | "s2" | "s3" *)
(* dir per session, from the folders: "live" ({id}/), "trash" *)
(* (.trash/{id}/), else "gone" once published, else "none" *)
(* index [id, op] per put or del line of index.jsonl, in file order; *)
(* a put that follows a put for the same id only updates the *)
(* summary and is left out, as are `opened` lines *)
(* pending, everDeleted what the code still owes, and a ghost *)
(***************************************************************************)
EXTENDS Catalog, TraceData, TLC
VARIABLE l
tvars == <<vars, l>>
TraceIds == {"s1", "s2", "s3"}
ASSUME TLCSet(1, 0)
Matches(e) ==
/\ dir' = e.dir
/\ index' = e.index
/\ pending' = e.pending
/\ everDeleted' = e.everDeleted
Act(e) ==
CASE e.event = "Publish" -> Publish(e.s)
[] e.event = "IndexPut" -> IndexPut(e.s)
[] e.event = "Trash" -> Trash(e.s)
[] e.event = "IndexDel" -> IndexDel(e.s)
[] e.event = "Purge" -> Purge(e.s)
[] e.event = "Heal" -> Heal(e.s)
[] e.event = "Crash" -> Crash
[] e.event = "Rebuild" -> Rebuild
TraceInit == Init /\ l = 1
TraceNext ==
/\ l <= Len(Trace)
/\ Act(Trace[l])
/\ Matches(Trace[l])
/\ l' = l + 1
TraceSpec == TraceInit /\ [][TraceNext]_tvars
Progress ==
IF l - 1 > TLCGet(1)
THEN TLCSet(1, l - 1) /\ PrintT(<<"matched_lines", l - 1>>)
ELSE TRUE
TraceIncomplete == l <= Len(Trace)
=============================================================================
@@ -0,0 +1,31 @@
------------------------------ MODULE ForkLog ------------------------------
(***************************************************************************)
(* A saved fork against its saved source (D5, D58). *)
(* *)
(* `Copied` is the fork's lines after line 1 up to its `forked_from.seq`, *)
(* the lines the manager copied; `Source` is the source's lines after *)
(* line 1. Both leave out snapshots, which a fork encodes again. Each line *)
(* is a record [k, h]: k its kind as TurnLifecycleLog names kinds, h a *)
(* hash of its exact bytes. Accepted when the copied lines are exactly the *)
(* source's first lines, byte for byte, and end at a turn boundary: every *)
(* turn they start, they also end. `NotWritten` then fails. *)
(***************************************************************************)
EXTENDS Naturals, Sequences, FiniteSets, SavedFork
VARIABLE checked
Starts(s) == {i \in 1..Len(s) : s[i].k = "start"}
Ends(s) == {i \in 1..Len(s) : s[i].k \in {"commit", "interrupt"}}
IsPrefix ==
/\ Len(Copied) <= Len(Source)
/\ \A i \in 1..Len(Copied) : Copied[i] = Source[i]
AtBoundary == Cardinality(Starts(Copied)) = Cardinality(Ends(Copied))
LogInit == checked = FALSE
LogNext == ~checked /\ IsPrefix /\ AtBoundary /\ checked' = TRUE
LogSpec == LogInit /\ [][LogNext]_checked
NotWritten == ~checked
=============================================================================
@@ -0,0 +1,8 @@
SPECIFICATION TraceSpec
CONSTANTS
Ids <- TraceIds
Blobs <- TraceBlobs
MaxLen <- TraceMaxLen
NoBlob <- TraceNoBlob
CONSTRAINT Progress
INVARIANT TraceIncomplete
@@ -0,0 +1,58 @@
----------------------------- MODULE ForkTrace -----------------------------
(***************************************************************************)
(* Trace validation for Fork.tla (blobs, fork at a boundary, delete). *)
(* *)
(* One trace line per action: *)
(* event Create | Start | Commit | Delete (with s) | PutBlob | Item *)
(* (with s and b) | Fork (with src, f, at) *)
(* s, src, f "s1" | "s2" | "s3"; b "b1" | "b2"; at the fork's S *)
(* st per session: "none" | "live" | "deleted" *)
(* lg per session, parsed from its log.jsonl: [k, b] for every *)
(* created, start, commit and item line; b is the item's blob *)
(* links per session: the blob files in its blobs/ folder *)
(* base per session: the prefix a fork copied, as the code copied it *)
(***************************************************************************)
EXTENDS Fork, TraceData, TLC
VARIABLE l
tvars == <<vars, l>>
TraceIds == {"s1", "s2", "s3"}
TraceBlobs == {"b1", "b2"}
TraceMaxLen == 32
TraceNoBlob == "none"
ASSUME TLCSet(1, 0)
Matches(e) ==
/\ st' = e.st
/\ lg' = e.lg
/\ links' = e.links
/\ base' = e.base
Act(e) ==
CASE e.event = "Create" -> Create(e.s)
[] e.event = "Start" -> Start(e.s)
[] e.event = "Commit" -> Commit(e.s)
[] e.event = "Delete" -> Delete(e.s)
[] e.event = "PutBlob" -> PutBlob(e.s, e.b)
[] e.event = "Item" -> Item(e.s, e.b)
[] e.event = "Fork" -> Fork(e.src, e.f, e.at)
TraceInit == Init /\ l = 1
TraceNext ==
/\ l <= Len(Trace)
/\ Act(Trace[l])
/\ Matches(Trace[l])
/\ l' = l + 1
TraceSpec == TraceInit /\ [][TraceNext]_tvars
Progress ==
IF l - 1 > TLCGet(1)
THEN TLCSet(1, l - 1) /\ PrintT(<<"matched_lines", l - 1>>)
ELSE TRUE
TraceIncomplete == l <= Len(Trace)
=============================================================================
@@ -0,0 +1,4 @@
SPECIFICATION TraceSpec
CONSTANT Ids <- TraceIds
CONSTRAINT Progress
INVARIANT TraceIncomplete
@@ -0,0 +1,58 @@
-------------------------- MODULE LifecycleTrace --------------------------
(***************************************************************************)
(* Trace validation for Lifecycle.tla (lazy creation and the first-turn *)
(* publish). *)
(* *)
(* One trace line per action: *)
(* event OpenNew | MkTmp | WriteLine1 | FsyncLog | Rename | *)
(* FsyncDirAndAck | Sweep (with s) | ProcessCrash | PowerLoss *)
(* s "s1" or "s2", for the per-session actions *)
(* step [s1 |-> ..., s2 |-> ...], how far each publish has got *)
(* where per session: "none" | "tmp" | "visible", from the folders *)
(* line1 per session: "none" | "cached" | "durable", from the log file *)
(* and the fault layer's record of what was synced *)
(* acked per session: the caller was told the first turn was saved *)
(***************************************************************************)
EXTENDS Lifecycle, Sequences, TraceData, TLC
VARIABLE l
tvars == <<vars, l>>
TraceIds == {"s1", "s2"}
ASSUME TLCSet(1, 0)
Matches(e) ==
/\ step' = e.step
/\ where' = e.where
/\ line1' = e.line1
/\ acked' = e.acked
Act(e) ==
CASE e.event = "OpenNew" -> OpenNew(e.s)
[] e.event = "MkTmp" -> MkTmp(e.s)
[] e.event = "WriteLine1" -> WriteLine1(e.s)
[] e.event = "FsyncLog" -> FsyncLog(e.s)
[] e.event = "Rename" -> Rename(e.s)
[] e.event = "FsyncDirAndAck" -> FsyncDirAndAck(e.s)
[] e.event = "Sweep" -> Sweep(e.s)
[] e.event = "ProcessCrash" -> ProcessCrash
[] e.event = "PowerLoss" -> PowerLoss
TraceInit == Init /\ l = 1
TraceNext ==
/\ l <= Len(Trace)
/\ Act(Trace[l])
/\ Matches(Trace[l])
/\ l' = l + 1
TraceSpec == TraceInit /\ [][TraceNext]_tvars
Progress ==
IF l - 1 > TLCGet(1)
THEN TLCSet(1, l - 1) /\ PrintT(<<"matched_lines", l - 1>>)
ELSE TRUE
TraceIncomplete == l <= Len(Trace)
=============================================================================
@@ -0,0 +1,145 @@
# Trace validation: checking the code against the specs
TLC cannot read Zig. Instead, the implementation writes one JSON line per
step it takes, and TLC checks that the whole sequence is a behavior the spec
allows. A trace is **accepted** only if some behavior of the spec produces
exactly the logged states, step by step. Otherwise it is **rejected**, and
`validate.sh` names the first step the spec does not allow.
The models in `..` check the design exhaustively within their bounds. Trace
validation checks that the real code stays inside the design on every run
we execute: unit tests, the standalone driver, and the `kill -9` and
machine-crash matrix.
## How it works
```text
Zig test build trace.ndjson to_tla.py TLC
------------------ -------------- --------- ---
each L1/L2 step -------> {"event": "Fsync", --> TraceData.tla --> SpecTrace.tla
emits one line "log_len": 2, ...} (Trace == <<...>>) EXTENDS Spec, TraceData
step l must be action
Trace[l].event, and the
next state must equal
Trace[l]'s fields
```
- `SpecTrace.tla` extends the spec unchanged. It adds a counter `l` over the
trace, and each TLC step must be the action named by `Trace[l].event`,
landing in a state whose variables equal that line's fields.
- Nondeterministic actions such as `PowerLoss` are resolved by the logged
state: TLC tries every choice and keeps the ones that match.
- The trick for the verdict: the invariant `TraceIncomplete == l <= Len(Trace)`
is violated exactly when the whole trace was matched, so TLC reporting a
violation means ACCEPTED. A state constraint prints the furthest line
matched, which names the rejected step.
## Files
| File | Purpose |
|---|---|
| `validate.sh SPEC trace.ndjson` | Runs the check in a temp directory. Exit 0 accepted, 1 rejected, 2 tool error |
| `to_tla.py` | Converts JSON lines into `TraceData.tla`. Lists become sequences, objects records, `{"$set": [...]}` sets |
| `SessionLogTrace.tla`, `.cfg` | Wrapper for `SessionLog.tla`. The event contract is in its header |
| `stand_in_log.py` | A small Python stand-in for L1, used only to self-test this tooling |
| `check_saved_logs.py ROOT...` | Checks logs fx saved (`{id}/log.jsonl` under each ROOT) after the fact: every log against `TurnLifecycle`, every parent with child lines against `Subagents`, and every fork against its source with `ForkLog`. `--selftest` requires 7 good and 9 planted logs to get their verdicts |
| `ForkLog.tla` | A fork's copied lines against its source's: accepted only when they are the source's first lines byte for byte (snapshots aside, which a fork encodes again) and end at a turn boundary |
| `TurnLifecycleLog.tla`, `SubagentsLog.tla` | Its wrappers. Only the disk is known, so TLC searches the process state between lines; accepted when some behavior writes exactly the saved lines |
| `check_wiring_traces.py LOG...` | Checks the adapter's `wiring` trace lines (a run with `FX_TRACE_SCOPES=wiring`) against `Wiring.tla`: each process's lines become Start, the host steps and Exit, or Crash when it stops without Exit. A log where two processes overlap is skipped. `--selftest` requires 6 good and 8 planted traces to get their verdicts, the fork cases included |
| `WiringTrace.tla` | Its wrapper. Accepted when every step is allowed in turn; the usage steps change nothing a traced step reads, so they are left out |
| `bun_tlc_check.sh FX_WORKTREE [FILE [PATTERN]]` | Runs fx's E2E tests under Bun with `FX_E2E_TLA_DIR` set, then checks every kept run with TLC: the `wiring` traces with `check_wiring_traces.py` and the saved logs with `check_saved_logs.py`. Exit 0 only when all are ACCEPTED and at least `MIN_FORKS` (default 1) fork steps and saved forks were checked; a skipped trace counts as a failure |
| `check_conversion.py --request R --report D` | Checks a v1-to-v2 conversion run against `V1Conversion.tla` (D59). The request comes from the pipeline harness suite `session-conversion-tlc`, which drives the real binary step by step (older-build v1 writes, opens, `fx sessions convert`, holds, damage, a crash mid-conversion) and observes each step's result on disk and through fx's own readers and listing. Exit 0 ACCEPTED only when TLC matched every observation, 1 REJECTED, 2 INCONCLUSIVE, 3 TOOL_ERROR. `--selftest` requires 6 good and 7 planted runs to get their verdicts |
| `V1ConversionTrace.tla` | Its wrapper. Each observation's step kind limits the model actions that may run before it; a request step makes exactly one request, and only a writer an earlier hold step left running may write during it |
| `selftest.sh` | Proves the tooling both accepts and rejects (below) |
## Self-test
`./selftest.sh` runs the stand-in three ways and requires these verdicts:
| Case | Required verdict |
|---|---|
| Correct stand-in (writes, fsync, a crash in the middle of a write, recovery, a machine crash that tears a line, recovery) | ACCEPTED, 15 steps |
| Recovery keeps a torn tail (planted bug) | REJECTED at step 8, the `Recover` |
| Good trace with `acked` changed from 2 to 3 on line 5 | REJECTED at step 5 |
Last run: 2026-09-27, TLC2 2.19. All three verdicts matched.
## The contract for the Zig trace hook
These rules apply to the test-only trace hook, `../../trace.zig`:
1. **Test builds only.** The hook is behind a build option and compiles to
nothing in release builds, so it adds no cost and no I/O to fx.
2. **One line per spec action,** named exactly like the action in the spec
(`BeginWrite`, `Fsync`, `Recover`, ...), with one field per spec variable
under the names in the wrapper header.
3. **Real state, not bookkeeping.** Fields that model the disk (line count,
torn tail, seq numbers, which folders exist) are computed by reading the
actual files after the step. Fields that model memory (status, acked, open
turn) come from the manager's own state. A trace built only from the
writer's counters would agree with itself and prove nothing.
4. **One order.** Steps inside a Session are emitted while holding the
Session mutex, so the trace order is the real order. Multi-process tests
(`WriterLock`) run through the standalone driver, which serializes the
steps of all processes into one trace.
5. **Crashes are logged by the driver.** A killed process cannot log its own
crash. After `kill -9` or a simulated machine crash, the driver appends the
`ProcessCrash` or `PowerLoss` line, reading the disk state itself.
6. **Every trace gets checked.** The driver runs `validate.sh` on every trace
it produces, and any rejection fails the run.
## Adding a wrapper for another spec
Copy `SessionLogTrace.tla` and change three things:
- `EXTENDS` names the new spec.
- `Matches(e)` compares every variable of that spec with the fields of `e`.
- `Act(e)` maps each event name to the spec's action. For an action with
parameters, such as `Write(k)` or `Open(p)`, the line carries the argument,
and `Act` passes it on, for example
`[] e.event = "Write" -> Write(e.kind)`.
Keep `TraceInit`, `TraceNext`, `Progress` and `TraceIncomplete` as they are,
and copy the `.cfg`, setting the spec's constants large enough for real
traces. Then add a stand-in case, or a real Zig test, that the new wrapper
must reject. A wrapper that has never rejected anything is not yet trusted.
Each of these wrappers has rejected its planted bug: `TurnLifecycleTrace`,
`WriterLockTrace`, `ResumeSnapshotTrace`, `LifecycleTrace`, `ForkTrace`,
`SubagentsTrace` and `CatalogTrace`.
## Limits
- **Guard with IF, never with `\/`, inside `Matches`.** Within an action,
TLC explores both sides of a disjunction, so
`Len(log') = 0 \/ log'[Len(log')].done` still indexes an empty sequence and
ends in a tool error. The first Zig traces (an `Fsync` and a `Recover` on an
empty log) found this in `SessionLogTrace.tla`; it now uses IF.
- **Give every TLC run its own `java.io.tmpdir`.** TLC unpacks `TLC.tla`,
`Naturals.tla` and the other standard modules there. With a shared
folder, parallel checks read each other's half-written copies and fail
with "Fatal errors while parsing". `validate.sh` points it into the
run's private work folder, and `check_saved_logs.py` and
`check_wiring_traces.py` give each parallel check its own (`JOBS`,
default 6).
- **A batch is one write; trace views end at the observed line.** The
manager writes each batch in one append, observed or not, so traced runs
and the crash matrix see production's write pattern (a process kill never
splits a batch). Its lines are observed one by one after the write, so
a tracer that reads the log bounds its view by the observed line's seq.
- **Only what runs is checked.** Trace validation checks the runs we
execute. That is why the driver's crash matrix matters: it decides which
code paths get traced.
- **The mapping from files to spec state must be right.** A wrong mapping can
hide a bug. Keep it small, and read the real files (rule 3).
- **Traces must fit the model's constants.** `SessionLogTrace.cfg` raises
`MaxLines` to 64 for that reason, and `TurnLifecycleTrace.tla` sets
`TraceMaxLen` to 4096 for the driver's host models, whose logs run to
hundreds of lines. A longer trace is rejected at the bound, so raise the
constant or split the trace. In a trace check the bound adds no states:
the trace fixes every step.
- **TLC 2.19 plus the Community Modules jar crashes**
(`NoClassDefFoundError: tlc2/value/impl/KSubsetValue`). That is why the
converter exists instead of `Json!ndJsonDeserialize`. Revisit when a TLC
release matches the modules jar.
@@ -0,0 +1,7 @@
SPECIFICATION TraceSpec
CONSTANTS
Vals <- TraceVals
MaxLen <- TraceMaxLen
NoVal <- TraceNoVal
CONSTRAINT Progress
INVARIANT TraceIncomplete
@@ -0,0 +1,51 @@
------------------------ MODULE ResumeSnapshotTrace ------------------------
(***************************************************************************)
(* Trace validation for ResumeSnapshot.tla (the fold and its snapshots). *)
(* *)
(* One trace line per action: *)
(* event Set (with v) | Start | Commit | Snapshot *)
(* v for Set: "v1" | "v2", the prefs value written *)
(* log every modeled line on disk, parsed from log.jsonl: *)
(* [k |-> "set" | "start" | "commit" | "snap", v, st]. For a *)
(* snapshot, st is read from the snapshot's own state object: *)
(* [pref, turns (committed), open]. Other kinds are outside this *)
(* spec and left out. *)
(* The Snapshot action requires st to equal the full fold at that point, *)
(* so a snapshot that stores the wrong state is rejected. *)
(***************************************************************************)
EXTENDS ResumeSnapshot, TraceData, TLC
VARIABLE l
tvars == <<vars, l>>
TraceVals == {"v1", "v2"}
TraceMaxLen == 64
TraceNoVal == "none"
ASSUME TLCSet(1, 0)
Matches(e) == log' = e.log
Act(e) ==
CASE e.event = "Set" -> Set(e.v)
[] e.event = "Start" -> Start
[] e.event = "Commit" -> Commit
[] e.event = "Snapshot" -> Snapshot
TraceInit == Init /\ l = 1
TraceNext ==
/\ l <= Len(Trace)
/\ Act(Trace[l])
/\ Matches(Trace[l])
/\ l' = l + 1
TraceSpec == TraceInit /\ [][TraceNext]_tvars
Progress ==
IF l - 1 > TLCGet(1)
THEN TLCSet(1, l - 1) /\ PrintT(<<"matched_lines", l - 1>>)
ELSE TRUE
TraceIncomplete == l <= Len(Trace)
=============================================================================
@@ -0,0 +1,4 @@
SPECIFICATION TraceSpec
CONSTANT MaxLines = 64
CONSTRAINT Progress
INVARIANT TraceIncomplete
@@ -0,0 +1,66 @@
------------------------- MODULE SessionLogTrace -------------------------
(***************************************************************************)
(* Trace validation for SessionLog.tla. *)
(* *)
(* TLC accepts an implementation trace only if some behavior of SessionLog *)
(* produces exactly the logged states, step by step. TraceData.tla is *)
(* generated from the implementation's JSON-lines trace by to_tla.py. *)
(* *)
(* One trace line per L1 step: *)
(* event BeginWrite | FinishWrite | Fsync | ProcessCrash | *)
(* PowerLoss | Recover *)
(* log_len lines in log.jsonl, counting a torn final line *)
(* tail_done whether the file ends in "\n" *)
(* seqs seq of every complete line, parsed from the file bytes *)
(* durable lines covered by the last fsync *)
(* acked lines acknowledged to callers as durable *)
(* status "open" | "writing" | "down" *)
(* log_len, tail_done and seqs must come from reading the real file, not *)
(* from the writer's own counters. *)
(***************************************************************************)
EXTENDS SessionLog, TraceData, TLC
VARIABLE l
tvars == <<vars, l>>
ASSUME TLCSet(1, 0)
\* IF, not \/: inside an action TLC explores both sides of a disjunction,
\* so `Len(log') = 0 \/ log'[Len(log')].done` would index an empty log.
Matches(e) ==
/\ Len(log') = e.log_len
/\ IF Len(log') = 0 THEN e.tail_done ELSE log'[Len(log')].done = e.tail_done
/\ Len(e.seqs) <= Len(log')
/\ \A i \in 1..Len(e.seqs) : log'[i].seq = e.seqs[i]
/\ durable' = e.durable
/\ acked' = e.acked
/\ status' = e.status
Act(e) ==
CASE e.event = "BeginWrite" -> BeginWrite
[] e.event = "FinishWrite" -> FinishWrite
[] e.event = "Fsync" -> Fsync
[] e.event = "ProcessCrash" -> ProcessCrash
[] e.event = "PowerLoss" -> PowerLoss
[] e.event = "Recover" -> Recover
TraceInit == Init /\ l = 1
TraceNext ==
/\ l <= Len(Trace)
/\ Act(Trace[l])
/\ Matches(Trace[l])
/\ l' = l + 1
TraceSpec == TraceInit /\ [][TraceNext]_tvars
\* State constraint: prints each new furthest trace line matched.
Progress ==
IF l - 1 > TLCGet(1)
THEN TLCSet(1, l - 1) /\ PrintT(<<"matched_lines", l - 1>>)
ELSE TRUE
\* Violated exactly when the whole trace was matched: TLC reporting this
\* invariant as violated means ACCEPTED.
TraceIncomplete == l <= Len(Trace)
=============================================================================
@@ -0,0 +1,31 @@
--------------------------- MODULE SubagentsLog ---------------------------
(***************************************************************************)
(* Checks a parent log fx saved against Subagents.tla, after the fact. *)
(* *)
(* Only the disk is known: the parent's child lines, in order (`Target`, *)
(* records [k, c, w, o]), and which children have a log at the end *)
(* (`FinalChildLog`), both from SavedChildren.tla. Threads, crashes, *)
(* reopens and when each child's log appeared are left to TLC. Accepted *)
(* when some behavior of the spec writes exactly these lines and ends *)
(* with those child logs: the invariant `NotWritten` then fails. *)
(* *)
(* check_saved_logs.py writes SavedChildren.tla and runs this. *)
(***************************************************************************)
EXTENDS Subagents, SavedChildren, TLC
ASSUME TLCSet(1, 0)
LogNext ==
/\ Next
/\ Len(plog') <= Len(Target)
/\ plog' = SubSeq(Target, 1, Len(plog'))
LogSpec == Init /\ [][LogNext]_vars
Progress ==
IF Len(plog) > TLCGet(1)
THEN TLCSet(1, Len(plog)) /\ PrintT(<<"matched_lines", Len(plog)>>)
ELSE TRUE
NotWritten == ~(plog = Target /\ childLog = FinalChildLog)
=============================================================================
@@ -0,0 +1,5 @@
SPECIFICATION TraceSpec
CONSTANT Children <- TraceChildren
CONSTANT MaxWork <- TraceMaxWork
CONSTRAINT Progress
INVARIANT TraceIncomplete
@@ -0,0 +1,58 @@
--------------------------- MODULE SubagentsTrace ---------------------------
(***************************************************************************)
(* Trace validation for Subagents.tla (children and parent reopen repair).*)
(* *)
(* One trace line per action: *)
(* event Spawn | ChildStart | ChildFinish | Cancel | RepairChild *)
(* (with c) | Crash | Reopen | RepairDone *)
(* c "c1" | "c2" *)
(* plog the parent's child lines, parsed from its log.jsonl: *)
(* [k |-> "spawned" | "finished", c, w (the child's work *)
(* number, counted from its spawn lines), o] *)
(* childLog per child: {child}/log.jsonl exists *)
(* thread, up, repairing the process's own state, as the code runs it *)
(***************************************************************************)
EXTENDS Subagents, TraceData, TLC
VARIABLE l
tvars == <<vars, l>>
TraceChildren == {"c1", "c2"}
TraceMaxWork == 2
ASSUME TLCSet(1, 0)
Matches(e) ==
/\ plog' = e.plog
/\ childLog' = e.childLog
/\ thread' = e.thread
/\ up' = e.up
/\ repairing' = e.repairing
Act(e) ==
CASE e.event = "Spawn" -> Spawn(e.c)
[] e.event = "ChildStart" -> ChildStart(e.c)
[] e.event = "ChildFinish" -> ChildFinish(e.c)
[] e.event = "Cancel" -> Cancel(e.c)
[] e.event = "RepairChild" -> RepairChild(e.c)
[] e.event = "Crash" -> Crash
[] e.event = "Reopen" -> Reopen
[] e.event = "RepairDone" -> RepairDone
TraceInit == Init /\ l = 1
TraceNext ==
/\ l <= Len(Trace)
/\ Act(Trace[l])
/\ Matches(Trace[l])
/\ l' = l + 1
TraceSpec == TraceInit /\ [][TraceNext]_tvars
Progress ==
IF l - 1 > TLCGet(1)
THEN TLCSet(1, l - 1) /\ PrintT(<<"matched_lines", l - 1>>)
ELSE TRUE
TraceIncomplete == l <= Len(Trace)
=============================================================================
@@ -0,0 +1,29 @@
------------------------- MODULE TurnLifecycleLog -------------------------
(***************************************************************************)
(* Checks a log fx saved against TurnLifecycle.tla, after the fact. *)
(* *)
(* Only the disk is known: the log's kinds, in order (`Target`, from *)
(* SavedLog.tla). The process state (status, fresh), and with it every *)
(* close, crash and reopen that wrote no line, is left to TLC. The log is *)
(* accepted when some behavior of the spec writes exactly these lines: *)
(* every step keeps the log a prefix of `Target`, and the invariant *)
(* `NotWritten` fails once the whole log is written. *)
(* *)
(* check_saved_logs.py writes SavedLog.tla and runs this. *)
(***************************************************************************)
EXTENDS TurnLifecycle, SavedLog, TLC
ASSUME TLCSet(1, 0)
LogNext == Next /\ Prefix(log', Target)
LogSpec == Init /\ [][LogNext]_vars
\* Prints the longest prefix written, which names the rejected line.
Progress ==
IF Len(log) > TLCGet(1)
THEN TLCSet(1, Len(log)) /\ PrintT(<<"matched_lines", Len(log)>>)
ELSE TRUE
NotWritten == log # Target
=============================================================================
@@ -0,0 +1,4 @@
SPECIFICATION TraceSpec
CONSTANT MaxLen <- TraceMaxLen
CONSTRAINT Progress
INVARIANT TraceIncomplete
@@ -0,0 +1,53 @@
------------------------ MODULE TurnLifecycleTrace ------------------------
(***************************************************************************)
(* Trace validation for TurnLifecycle.tla (turn rules, close, reopen). *)
(* *)
(* One trace line per action: *)
(* event Create | Write (with kind) | Close | Crash | Reopen *)
(* kind for Write: start | item | compacted | commit | interrupt | set *)
(* log the kinds on disk, in order, parsed from log.jsonl; snapshot *)
(* and child lines are outside this spec and left out *)
(* status "new" | "open" | "closed" | "down" *)
(* fresh TRUE from a reopen until the next turn starts *)
(***************************************************************************)
EXTENDS TurnLifecycle, TraceData, TLC
VARIABLE l
tvars == <<vars, l>>
\* Only bounds the log a trace may build; the trace fixes every step, so
\* a large bound adds no states. The driver's host models run to hundreds
\* of lines.
TraceMaxLen == 4096
ASSUME TLCSet(1, 0)
Matches(e) ==
/\ log' = e.log
/\ status' = e.status
/\ fresh' = e.fresh
Act(e) ==
CASE e.event = "Create" -> Create
[] e.event = "Write" -> Write(e.kind)
[] e.event = "Close" -> Close
[] e.event = "Crash" -> Crash
[] e.event = "Reopen" -> Reopen
TraceInit == Init /\ l = 1
TraceNext ==
/\ l <= Len(Trace)
/\ Act(Trace[l])
/\ Matches(Trace[l])
/\ l' = l + 1
TraceSpec == TraceInit /\ [][TraceNext]_tvars
Progress ==
IF l - 1 > TLCGet(1)
THEN TLCSet(1, l - 1) /\ PrintT(<<"matched_lines", l - 1>>)
ELSE TRUE
TraceIncomplete == l <= Len(Trace)
=============================================================================
@@ -0,0 +1,108 @@
--------------------------- MODULE V1ConversionTrace ---------------------------
(***************************************************************************)
(* Trace validation for V1Conversion.tla from what real fx runs leave. *)
(* *)
(* A driver runs real fx processes one step at a time and observes the *)
(* result after each (`Obs`, from ConversionData.tla), a record with *)
(* k the step: create | write | hold | release (older-build v1 *)
(* writers), open | all | crash (a request, the last one killed *)
(* mid-request), open2 (two processes opening the same root at *)
(* once), damage, fresh (a new v2 session), list *)
(* r the requested root ("none" for other steps) *)
(* ans the request's answer: opened | converted | busy | unreadable *)
(* | notfound, or none; ans2 the second process's, for open2 *)
(* v1 [Ids -> version]: each v1 member's distinct contents in order *)
(* of first appearance, 0 absent *)
(* pub [AllIds -> Status]: each id in v2 *)
(* base [AllIds -> version]: for a converted id, the v1 version its *)
(* copy equals, -1 when it equals none *)
(* extra [AllIds -> Nat]: v2 turns after the copied ones *)
(* dirty conversion leftovers in .tmp *)
(* listed the ids the listing shows *)
(* bad [Ids -> BOOLEAN]: members the driver made unreadable *)
(* held the members a still-running older build has open *)
(* denied the driver made the file system refuse deletes in this step *)
(* Between observations, only the actions of that step's kind may run, *)
(* any number of them; a request step makes exactly its requests, by *)
(* process p1 (and p2 for open2). Accepted when some behavior of *)
(* V1Conversion passes through every observation in order: `NotDone` *)
(* then fails. *)
(***************************************************************************)
EXTENDS V1Conversion, ConversionData, Sequences, TLC
VARIABLE l
tvars == <<vars, l>>
ASSUME TLCSet(1, 0)
P1 == "p1"
P2 == "p2"
\* The requests observation `j` makes.
Made(o) == CASE o.k = "open2" -> 2 [] o.k \in {"open", "all", "crash"} -> 1 [] OTHER -> 0
RECURSIVE ReqCount(_)
ReqCount(n) == IF n = 0 THEN 0 ELSE Made(Obs[n]) + ReqCount(n - 1)
\* A refused delete is a fault only a step that injected it may show.
ConvAllowed(p, o) == ConvWork(p) \/ (o.denied /\ ConvRefused(p))
\* During a request, an older build may write only if an earlier hold step
\* left it holding the session; it cannot take a lock there.
HolderStep(i) == WBegin(i) \/ WEnd(i) \/ WMark(i)
\* The actions a step of kind `o.k` may take, at observation index `n`.
Hidden(o, n) ==
CASE o.k \in {"create", "write", "hold", "release"} ->
\E i \in Ids : WriterStep(i)
[] o.k = "damage" -> \E i \in Ids : Damage(i)
[] o.k = "fresh" -> \E i \in NewIds : FCreate(i) \/ FAppend(i)
[] o.k \in {"open", "all"} ->
\/ reqs < ReqCount(n) /\ Request(P1, o.k, o.r)
\/ ConvAllowed(P1, o)
\/ \E i \in Ids : HolderStep(i)
\/ req[P1] = NoReq /\ reqs = ReqCount(n) /\ \E i \in AllIds : FAppend(i)
[] o.k = "open2" ->
\/ reqs = ReqCount(n) - 2 /\ Request(P1, "open", o.r)
\/ reqs = ReqCount(n) - 1 /\ Request(P2, "open", o.r)
\/ \E p \in {P1, P2} : ConvAllowed(p, o)
\/ \E i \in Ids : HolderStep(i)
\/ req[P1] = NoReq /\ req[P2] = NoReq /\ reqs = ReqCount(n) /\ \E i \in AllIds : FAppend(i)
[] o.k = "crash" ->
\/ reqs < ReqCount(n) /\ \E kk \in {"open", "all"} : Request(P1, kk, o.r)
\/ ConvAllowed(P1, o) \/ CCrash(P1)
\/ \E i \in Ids : HolderStep(i)
[] OTHER -> FALSE
Match(o, n) ==
/\ v1ver = o.v1
/\ pub = o.pub
/\ \A i \in AllIds : pub[i] = "import" => base[i] = o.base[i]
/\ extra = o.extra
/\ Dirty = o.dirty
/\ Listed = o.listed
/\ bad = o.bad
/\ {i \in Ids : v1lock[i] = "writer"} = o.held
/\ reqs = ReqCount(n)
/\ \A p \in Procs : req[p] = NoReq /\ conv[p] = Idle
/\ o.k \in {"open", "all"} => out[P1].ans = o.ans
/\ o.k = "open2" =>
\/ out[P1].ans = o.ans /\ out[P2].ans = o.ans2
\/ out[P1].ans = o.ans2 /\ out[P2].ans = o.ans
/\ o.k = "crash" => out[P1].ans = "none"
TraceInit == Init /\ l = 1
TraceNext ==
/\ l <= Len(Obs)
/\ \/ Hidden(Obs[l], l) /\ UNCHANGED l
\/ Match(Obs[l], l) /\ l' = l + 1 /\ UNCHANGED vars
TraceSpec == TraceInit /\ [][TraceNext]_tvars
Progress ==
IF l - 1 > TLCGet(1)
THEN TLCSet(1, l - 1) /\ PrintT(<<"matched_lines", l - 1>>)
ELSE TRUE
NotDone == l <= Len(Obs)
=============================================================================
@@ -0,0 +1,55 @@
------------------------------ MODULE WiringTrace ------------------------------
(***************************************************************************)
(* Trace validation for Wiring.tla, from fx's `wiring` trace lines. *)
(* *)
(* One trace line per step the adapter logs, in order (`Trace`, from *)
(* WiringData.tla): a record [a, s, e] with *)
(* a Start | Open | BeginTurn | EndTurn | Spawn | ChildDone | Close *)
(* | Fork | Exit | Crash *)
(* s the session, "s1".. (Open, Close, and Fork's new session), else *)
(* "none" *)
(* e "commit" | "interrupt" (EndTurn only), else "none" *)
(* Start and Exit are the store's first open and last close in a process *)
(* (a process whose tracing starts after its store opened starts at its *)
(* first line); *)
(* Crash is added by check_wiring_traces.py when a process's lines stop *)
(* without Exit. The usage steps (the marker, charges, checkpoints) happen *)
(* in code the adapter does not see; they change only `usageSplit` and *)
(* `marker`, which no traced step reads, so they are left out. Accepted *)
(* when every line is an allowed step in turn: `NotDone` then fails. *)
(***************************************************************************)
EXTENDS Wiring, WiringData, Sequences, TLC
VARIABLE l
tvars == <<vars, l>>
ASSUME TLCSet(1, 0)
Act(e) ==
CASE e.a = "Start" -> Start("v2")
[] e.a = "Open" -> Open(e.s)
[] e.a = "BeginTurn" -> BeginTurn
[] e.a = "EndTurn" -> EndTurn(e.e)
[] e.a = "Spawn" -> Spawn
[] e.a = "ChildDone" -> ChildDone
[] e.a = "Close" -> Close(e.s)
[] e.a = "Fork" -> Fork(e.s)
[] e.a = "Exit" -> Exit
[] e.a = "Crash" -> Crash
TraceInit == Init /\ l = 1
TraceNext ==
/\ l <= Len(Trace)
/\ Act(Trace[l])
/\ l' = l + 1
TraceSpec == TraceInit /\ [][TraceNext]_tvars
Progress ==
IF l - 1 > TLCGet(1)
THEN TLCSet(1, l - 1) /\ PrintT(<<"matched_lines", l - 1>>)
ELSE TRUE
NotDone == l <= Len(Trace)
=============================================================================
@@ -0,0 +1,8 @@
SPECIFICATION TraceSpec
CONSTANTS
Procs <- TraceProcs
Threads <- TraceThreads
MaxLines <- TraceMaxLines
None <- TraceNone
CONSTRAINT Progress
INVARIANT TraceIncomplete
@@ -0,0 +1,62 @@
-------------------------- MODULE WriterLockTrace --------------------------
(***************************************************************************)
(* Trace validation for WriterLock.tla (flock plus the Session mutex). *)
(* *)
(* One trace line per action: *)
(* event Open | Close | Crash | Restart (with p) | *)
(* Acquire | WriteAndRelease (with p and t) *)
(* p, t "p1" | "p2" and "t1" | "t2" *)
(* holder the process holding the flock, or "none" *)
(* alive, opened, nextSeq, mutex per process *)
(* file [seq, by] for each line written since the trace began, *)
(* parsed from log.jsonl: seq counts from 1 at the first traced *)
(* line, `by` is the writer named in the line. `closed` lines *)
(* are bookkeeping outside this spec and are left out, and the *)
(* seqs after them shift down, so a skipped or duplicated seq *)
(* still shows as a mismatch. *)
(***************************************************************************)
EXTENDS WriterLock, TraceData, TLC
VARIABLE l
tvars == <<vars, l>>
TraceProcs == {"p1", "p2"}
TraceThreads == {"t1", "t2"}
TraceMaxLines == 64
TraceNone == "none"
ASSUME TLCSet(1, 0)
Matches(e) ==
/\ holder' = e.holder
/\ alive' = e.alive
/\ opened' = e.opened
/\ nextSeq' = e.nextSeq
/\ mutex' = e.mutex
/\ file' = e.file
Act(e) ==
CASE e.event = "Open" -> Open(e.p)
[] e.event = "Close" -> Close(e.p)
[] e.event = "Crash" -> Crash(e.p)
[] e.event = "Restart" -> Restart(e.p)
[] e.event = "Acquire" -> Acquire(e.p, e.t)
[] e.event = "WriteAndRelease" -> WriteAndRelease(e.p, e.t)
TraceInit == Init /\ l = 1
TraceNext ==
/\ l <= Len(Trace)
/\ Act(Trace[l])
/\ Matches(Trace[l])
/\ l' = l + 1
TraceSpec == TraceInit /\ [][TraceNext]_tvars
Progress ==
IF l - 1 > TLCGet(1)
THEN TLCSet(1, l - 1) /\ PrintT(<<"matched_lines", l - 1>>)
ELSE TRUE
TraceIncomplete == l <= Len(Trace)
=============================================================================
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env bash
# Runs fx's E2E tests under Bun with traces kept, then checks every run with
# TLC: each process's `wiring` trace against Wiring.tla, and each saved log
# against TurnLifecycle, Subagents and, for a fork, ForkLog. Exits 0 only when
# every check is ACCEPTED and the runs held at least MIN_FORKS fork steps.
#
# bun_tlc_check.sh FX_WORKTREE [TEST_FILE [TEST_NAME_PATTERN]]
#
# FX_WORKTREE is an fx checkout whose zig-out/bin/fx is built. MIN_FORKS
# (default 1) guards against a run that never reached the code under check.
set -euo pipefail
here="$(cd "$(dirname "$0")" && pwd)"
fx="${1:?usage: bun_tlc_check.sh FX_WORKTREE [TEST_FILE [TEST_NAME_PATTERN]]}"
file="${2:-sessions-v2.test.ts}"
pattern="${3:-fork}"
min_forks="${MIN_FORKS:-1}"
[ -x "$fx/zig-out/bin/fx" ] || { echo "build fx first: $fx/zig-out/bin/fx" >&2; exit 2; }
out="$(mktemp -d "${TMPDIR:-/tmp}/fx-bun-tlc.XXXXXX")"
echo "runs kept in $out"
(cd "$fx/tests/e2e" && env -u AI_GATEWAY_API_KEY -u AI_GATEWAY_TEST_API_KEY \
FX_E2E_TLA_DIR="$out" bun test "$file" -t "$pattern") || {
echo "bun: the tests failed; TLC not run" >&2
exit 1
}
traces=()
while IFS= read -r f; do traces+=("$f"); done < <(find "$out" -name '*.wiring.log' | sort)
roots=()
while IFS= read -r d; do roots+=("$d"); done < <(find "$out" -type d -name 'sessions-v2' | sort)
[ "${#traces[@]}" -gt 0 ] || { echo "no wiring traces kept: is FX_E2E_TLA_DIR honored?" >&2; exit 1; }
[ "${#roots[@]}" -gt 0 ] || { echo "no saved logs kept" >&2; exit 1; }
forks="$(cat "${traces[@]}" | grep -c 'action=Fork ' || true)"
echo "traces ${#traces[@]}, saved roots ${#roots[@]}, fork steps $forks"
[ "$forks" -ge "$min_forks" ] || { echo "fewer than $min_forks fork steps reached TLC" >&2; exit 1; }
status=0
echo "== wiring traces against Wiring.tla"
wiring="$(python3 "$here/check_wiring_traces.py" "${traces[@]}")" || status=1
echo "$wiring"
if echo "$wiring" | grep -q -E '^(REJECTED|SKIPPED|TOOL ERROR) +/'; then status=1; fi
# An empty trace is right only for a run on v1, which saves no v2 sessions.
while IFS= read -r path; do
[ -n "$path" ] || continue
if [ -d "$(dirname "$path")/sessions-v2" ]; then
echo "a v2 run left an empty trace: $path" >&2
status=1
fi
done < <(echo "$wiring" | sed -n -E 's/^EMPTY +(.*): no wiring lines$/\1/p')
echo "$wiring" | grep -q -E '^ACCEPTED +[1-9]' || { echo "no trace was accepted" >&2; status=1; }
echo "== saved logs against TurnLifecycle, Subagents and ForkLog"
saved="$(python3 "$here/check_saved_logs.py" "${roots[@]}")" || status=1
echo "$saved"
fork_jobs="$(echo "$saved" | awk '$1 == "fork" && $2 == "ACCEPTED" {print $3}')"
[ "${fork_jobs:-0}" -ge "$min_forks" ] || { echo "fewer than $min_forks saved forks checked" >&2; status=1; }
[ "$status" -eq 0 ] && echo "ALL ACCEPTED" || echo "NOT ALL ACCEPTED"
exit "$status"
@@ -0,0 +1,311 @@
#!/usr/bin/env python3
"""Check what real fx runs did against V1Conversion.tla, with TLC.
check_conversion.py --request REQUEST.json --report DIR
check_conversion.py --selftest
REQUEST is written by the Bun driver: {"version": 1, "ids": {"root": ID,
"child": ID or null, "fresh": [ID, ...]}, "observations": [...]}, one
observation per real step, as V1ConversionTrace.tla describes. The verdict
is printed as JSON and kept in DIR with TLC's log and its generated inputs:
ACCEPTED (exit 0) only when TLC reached its sentinel with every observation
matched; REJECTED (1) when TLC ran to the end without matching them all;
INCONCLUSIVE (2) for malformed or empty evidence; TOOL_ERROR (3) otherwise.
"""
import argparse
import hashlib
import json
import os
import pwd
import re
import shutil
import subprocess
import sys
from pathlib import Path
HERE = Path(__file__).resolve().parent
TLA = HERE.parent
JAVA = os.environ.get("JAVA", "/opt/homebrew/opt/openjdk/bin/java")
# The account's home, not $HOME: drivers run fx under a disposable HOME.
JAR = os.environ.get("TLA_JAR", os.path.join(pwd.getpwuid(os.getuid()).pw_dir, ".local/share/tla/tla2tools.jar"))
KINDS = {"create", "write", "hold", "release", "open", "all", "crash", "open2", "damage", "fresh", "list"}
REQUESTS = {"open", "all", "crash", "open2"}
ANSWERS = {"none", "opened", "converted", "busy", "unreadable", "notfound"}
STATUS = {"none", "import", "fresh"}
EXIT = {"ACCEPTED": 0, "REJECTED": 1, "INCONCLUSIVE": 2, "TOOL_ERROR": 3}
class Inconclusive(Exception):
pass
def tla_str(s):
return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"'
def tla_fun(mapping, render):
return "(" + " @@ ".join(f"{tla_str(k)} :> {render(v)}" for k, v in mapping.items()) + ")"
def model_ids(ids):
"""Real id -> model id: the root "r", the child "c", fresh sessions "n1".."""
root, child, fresh = ids.get("root"), ids.get("child"), ids.get("fresh", [])
if not isinstance(root, str) or not root:
raise Inconclusive("the request names no root id")
names = {root: "r"}
if child:
names[child] = "c"
for n, fid in enumerate(fresh, 1):
names[fid] = f"n{n}"
if len(names) != 1 + bool(child) + len(fresh):
raise Inconclusive("ids repeat")
return names
def model_obs(request):
names = model_ids(request.get("ids", {}))
obs = request.get("observations")
if not isinstance(obs, list) or not obs:
raise Inconclusive("no observations")
if not any(o.get("k") in REQUESTS for o in obs):
raise Inconclusive("no request step was observed")
member = ["r", "c"]
every = member + sorted(v for v in names.values() if v.startswith("n"))
out = []
for n, o in enumerate(obs, 1):
try:
k, ans, ans2 = o["k"], o.get("ans", "none"), o.get("ans2", "none")
if k not in KINDS or ans not in ANSWERS or ans2 not in ANSWERS:
raise Inconclusive(f"observation {n}: unknown step or answer")
r = names.get(o.get("r"), "none") if k in REQUESTS else "none"
if k in REQUESTS and r != "r":
raise Inconclusive(f"observation {n}: a request names no known root")
def by_model(field, default, check):
got = {names[i]: v for i, v in o[field].items() if i in names}
if len(got) != len(o[field]):
raise Inconclusive(f"observation {n}: {field} names an unknown id")
for v in got.values():
check(v)
return got, default
def nat(v):
if not isinstance(v, int) or v < -1:
raise Inconclusive(f"observation {n}: a count is not an integer")
def status(v):
if v not in STATUS:
raise Inconclusive(f"observation {n}: unknown v2 status {v!r}")
def flag(v):
if not isinstance(v, bool):
raise Inconclusive(f"observation {n}: a flag is not a boolean")
v1, _ = by_model("v1", 0, nat)
pub, _ = by_model("pub", "none", status)
base, _ = by_model("base", 0, nat)
extra, _ = by_model("extra", 0, nat)
bad, _ = by_model("bad", False, flag)
listed, held, denied = o["listed"], o.get("held", []), o.get("denied", False)
if not isinstance(o["dirty"], bool) or not isinstance(listed, list) or not isinstance(held, list) \
or not isinstance(denied, bool):
raise Inconclusive(f"observation {n}: dirty, listed, held or denied is malformed")
if denied and k not in REQUESTS:
raise Inconclusive(f"observation {n}: deletes can only be refused during a request")
if any(names.get(i) not in member for i in held):
raise Inconclusive(f"observation {n}: held names an id that is not a v1 member")
unknown = [i for i in listed if i not in names]
out.append({
"k": k, "r": r, "ans": ans, "ans2": ans2,
"v1": {i: v1.get(i, 0) for i in member},
"pub": {i: pub.get(i, "none") for i in every},
"base": {i: base.get(i, 0) for i in every},
"extra": {i: extra.get(i, 0) for i in every},
"bad": {i: bad.get(i, False) for i in member},
"dirty": o["dirty"],
"denied": denied,
"held": sorted(names[i] for i in held),
# An id the run does not know can never match the model's listing.
"listed": sorted({names[i] for i in listed if i in names} | ({"unknown"} if unknown else set())),
})
except KeyError as missing:
raise Inconclusive(f"observation {n}: missing {missing}")
return out, [v for v in every if v.startswith("n")]
def data_module(obs):
def rec(o):
return ("[k |-> " + tla_str(o["k"]) + ", r |-> " + tla_str(o["r"]) + ", ans |-> " + tla_str(o["ans"])
+ ", ans2 |-> " + tla_str(o["ans2"])
+ ", v1 |-> " + tla_fun(o["v1"], str) + ", pub |-> " + tla_fun(o["pub"], tla_str)
+ ", base |-> " + tla_fun(o["base"], str) + ", extra |-> " + tla_fun(o["extra"], str)
+ ", bad |-> " + tla_fun(o["bad"], lambda b: "TRUE" if b else "FALSE")
+ ", dirty |-> " + ("TRUE" if o["dirty"] else "FALSE")
+ ", denied |-> " + ("TRUE" if o["denied"] else "FALSE")
+ ", held |-> {" + ", ".join(tla_str(i) for i in o["held"]) + "}"
+ ", listed |-> {" + ", ".join(tla_str(i) for i in o["listed"]) + "}]")
body = ",\n ".join(rec(o) for o in obs)
return f"---- MODULE ConversionData ----\nEXTENDS TLC, Integers, FiniteSets\nObs == <<\n {body}\n>>\n====\n"
def config(obs, fresh):
kinds = [o["k"] for o in obs]
max_v = max([1] + [v for o in obs for v in o["v1"].values()])
max_extra = max([1] + [v for o in obs for v in o["extra"].values()])
writes = sum(k in {"create", "write", "hold"} for k in kinds) + 1
crashes = sum(k in {"crash", "release"} for k in kinds) + 3 * sum(o["denied"] for o in obs) + 1
reqs = sum(2 if k == "open2" else 1 for k in kinds if k in REQUESTS)
fresh_set = "{" + ", ".join(tla_str(f) for f in fresh) + "}"
return (f'CONSTANTS\n Procs = {{"p1", "p2"}}\n RootId = "r"\n ChildId = "c"\n NewIds = {fresh_set}\n None = None\n'
f" Torn = Torn\n MaxV = {max_v}\n MaxLocks = {writes}\n MaxExtra = {max_extra}\n"
f" MaxCrash = {crashes}\n MaxReq = {reqs}\n AvoidV1Ids = TRUE\n"
"SPECIFICATION TraceSpec\nINVARIANT NotDone\nCONSTRAINT Progress\n")
def sha256(path):
return hashlib.sha256(Path(path).read_bytes()).hexdigest()
def run_tlc(obs, fresh, work):
work.mkdir(parents=True)
(work / "tmp").mkdir()
for name in ("V1Conversion.tla",):
shutil.copyfile(TLA / name, work / name)
shutil.copyfile(HERE / "V1ConversionTrace.tla", work / "V1ConversionTrace.tla")
(work / "ConversionData.tla").write_text(data_module(obs))
(work / "V1ConversionTrace.cfg").write_text(config(obs, fresh))
command = [JAVA, "-XX:+UseParallelGC", f"-Djava.io.tmpdir={work / 'tmp'}", "-cp", JAR, "tlc2.TLC",
"-workers", "1", "-deadlock", "-metadir", str(work / "states"),
"-config", "V1ConversionTrace.cfg", "V1ConversionTrace"]
with (work / "tlc.log").open("w") as log:
code = subprocess.run(command, cwd=work, stdout=log, stderr=subprocess.STDOUT, timeout=600).returncode
text = (work / "tlc.log").read_text(errors="replace")
matched = max([int(m) for m in re.findall(r'"matched_lines", (\d+)', text)] or [0])
expected = len(obs)
if code == 12 and "Invariant NotDone is violated" in text and matched == expected:
verdict = "ACCEPTED"
elif code == 0 and "No error has been found" in text and matched < expected:
verdict = "REJECTED"
else:
verdict = "TOOL_ERROR"
shutil.rmtree(work / "states", ignore_errors=True)
shutil.rmtree(work / "tmp", ignore_errors=True)
return {"verdict": verdict, "matched": matched, "expected": expected, "exitCode": code,
"log": str(work / "tlc.log"),
"modelHashes": {n: sha256(work / n) for n in ("V1Conversion.tla", "V1ConversionTrace.tla")}}
def verify(request, report):
if request.get("version") != 1:
raise Inconclusive("version 1 required")
obs, fresh = model_obs(request)
result = run_tlc(obs, fresh, report / "tlc")
if result["verdict"] == "REJECTED":
step = obs[result["matched"]]
result["detail"] = (f"observations 1..{result['matched']} match; observation {result['matched'] + 1} "
f"({step['k']}, answer {step['ans']}) is not a behavior of V1Conversion")
return result
def selftest():
"""Planted observation sequences: each must get its verdict."""
ids = {"root": "R", "child": None, "fresh": []}
def ob(k, v1=0, pub="none", base=0, extra=0, ans="none", dirty=False, listed=None, bad=False, r="R", held=(),
denied=False, ans2="none"):
return {"k": k, "r": r, "ans": ans, "ans2": ans2, "v1": {"R": v1}, "pub": {"R": pub}, "base": {"R": base},
"extra": {"R": extra}, "bad": {"R": bad}, "dirty": dirty, "held": list(held),
"listed": ["R"] if listed is None else listed, "denied": denied}
made = ob("create", v1=1)
kept = ob("open", v1=1, pub="import", base=1, ans="opened", denied=True)
cases = [
("an open converts the v1 session and deletes it, then a turn lands in v2",
[made, ob("open", pub="import", base=1, extra=1, ans="opened")], "ACCEPTED"),
("convert-all, then an open of the converted session",
[made, ob("all", pub="import", base=1, ans="converted"),
ob("open", pub="import", base=1, ans="opened")], "ACCEPTED"),
("busy while an older build holds it, opened once released",
[made, ob("hold", v1=1, held=["R"]), ob("open", v1=1, ans="busy", held=["R"]), ob("release", v1=1),
ob("open", pub="import", base=1, ans="opened")], "ACCEPTED"),
("unreadable: refused, nothing converted or deleted",
[made, ob("damage", v1=1, bad=True), ob("open", v1=1, bad=True, ans="unreadable")], "ACCEPTED"),
("a refused delete leaves the folder; the next open deletes it",
[made, kept, ob("open", pub="import", base=1, ans="opened")], "ACCEPTED"),
("an older build changes a leftover; the next open keeps it, and v2 keeps its copy",
[made, kept, ob("write", v1=2, pub="import", base=1),
ob("open", v1=2, pub="import", base=1, ans="opened")], "ACCEPTED"),
("a crash mid-conversion leaves staging; the next open sweeps it",
[made, ob("crash", v1=1, dirty=True), ob("open", pub="import", base=1, ans="opened")], "ACCEPTED"),
("two processes open it at once: one converts, both open the copy",
[made, ob("open2", pub="import", base=1, ans="opened", ans2="opened")], "ACCEPTED"),
("planted: of two processes opening it at once, one calls it unreadable",
[made, ob("open2", pub="import", base=1, ans="opened", ans2="unreadable")], "REJECTED"),
("planted: a live v1 session reported missing",
[made, ob("open", v1=1, ans="notfound")], "REJECTED"),
("planted: the copy matches no v1 version",
[made, ob("open", pub="import", base=-1, ans="opened")], "REJECTED"),
("planted: busy with no holder",
[made, ob("open", v1=1, ans="busy")], "REJECTED"),
("planted: the listing hides an unconverted v1 session",
[made, ob("list", v1=1, listed=[]), ob("open", pub="import", base=1, ans="opened")], "REJECTED"),
("planted: unreadable, yet half converted",
[made, ob("damage", v1=1, bad=True), ob("open", v1=1, bad=True, dirty=True, ans="unreadable")], "REJECTED"),
("planted: an older build's later write reaches the v2 copy",
[made, kept, ob("write", v1=2, pub="import", base=2)], "REJECTED"),
("planted: converted with no request",
[made, ob("list", v1=1, pub="import", base=1), ob("open", pub="import", base=1, ans="opened")], "REJECTED"),
("planted: the v1 folder stays though no delete was refused",
[made, ob("open", v1=1, pub="import", base=1, ans="opened")], "REJECTED"),
("planted: a leftover an older build changed is deleted",
[made, kept, ob("write", v1=2, pub="import", base=1), ob("open", pub="import", base=1, ans="opened")],
"REJECTED"),
("planted: the v1 folder deleted before its copy was published",
[made, ob("crash", dirty=True)], "REJECTED"),
]
bad = 0
root = Path(os.environ.get("TMPDIR", "/tmp")) / f"check-conversion-selftest-{os.getpid()}"
for n, (label, observations, want) in enumerate(cases, 1):
try:
got = verify({"version": 1, "ids": ids, "observations": observations}, root / str(n))
verdict = got["verdict"]
except Inconclusive as why:
verdict = f"INCONCLUSIVE ({why})"
ok = verdict == want
bad += not ok
print(f"{'ok ' if ok else 'FAIL'} {label}: {verdict}, wanted {want}")
print(f"selftest: {len(cases) - bad}/{len(cases)} verdicts as required")
if not bad:
shutil.rmtree(root, ignore_errors=True)
return 1 if bad else 0
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--request", type=Path)
parser.add_argument("--report", type=Path)
parser.add_argument("--selftest", action="store_true")
args = parser.parse_args()
if args.selftest:
return selftest()
if not args.request or not args.report:
parser.error("--request and --report are required")
report = args.report.resolve()
report.mkdir(parents=True, exist_ok=False)
try:
raw = args.request.read_bytes()
(report / "request.json").write_bytes(raw)
result = verify(json.loads(raw), report)
except Inconclusive as why:
result = {"verdict": "INCONCLUSIVE", "detail": str(why)}
except (OSError, ValueError, subprocess.SubprocessError) as error:
result = {"verdict": "TOOL_ERROR", "detail": str(error)}
result["version"] = 1
result["report"] = str(report)
(report / "verdict.json").write_text(json.dumps(result, indent=2) + "\n")
print(json.dumps(result))
return EXIT[result["verdict"]]
if __name__ == "__main__":
sys.exit(main())
@@ -0,0 +1,298 @@
#!/usr/bin/env python3
"""Checks logs fx saved against TurnLifecycle.tla and Subagents.tla.
usage: check_saved_logs.py ROOT... each ROOT holds session folders
({id}/log.jsonl), as ~/.fx/sessions/v2
check_saved_logs.py --selftest planted logs: each must get its verdict
JOBS=N runs N TLC checks at once (default 6).
Every session's log is checked against TurnLifecycle (TurnLifecycleLog.tla);
a log with child lines is also checked against Subagents (SubagentsLog.tla),
with each child's log existence read at the end. Only the disk is known, so
TLC searches the process state in between (see the two wrappers). A torn
tail is left out, as the manager reads it. A log that does not parse is
reported as unreadable, not checked. Exit 0 when every readable log is
accepted.
"""
import hashlib
import json
import os
import pwd
import re
import shutil
import subprocess
import sys
import tempfile
from concurrent.futures import ThreadPoolExecutor
HERE = os.path.dirname(os.path.abspath(__file__))
TLA = os.path.dirname(HERE)
JAVA = os.environ.get("JAVA", "/opt/homebrew/opt/openjdk/bin/java")
# The account's home, not $HOME: drivers run fx under a disposable HOME.
JAR = os.environ.get("TLA_JAR", os.path.join(pwd.getpwuid(os.getuid()).pw_dir, ".local/share/tla/tla2tools.jar"))
TURN_KINDS = {
"session_created": "created",
"set": "set",
"turn_started": "start",
"item": "item",
"compacted": "compacted",
"turn_committed": "commit",
"turn_interrupted": "interrupt",
"closed": "closed",
}
CHILD_KINDS = {"child_spawned", "child_finished"}
OUTSIDE = {"snapshot"} | CHILD_KINDS
class Unreadable(Exception):
pass
def read_lines(path, raw=False):
"""The log's whole lines; a torn tail is left out. With `raw`, each line
comes with its exact bytes."""
data = open(path, "rb").read()
end = data.rfind(b"\n")
out = []
for n, line in enumerate(data[: end + 1].split(b"\n")[:-1], 1):
try:
value = json.loads(line)
except ValueError:
raise Unreadable(f"line {n} is not JSON")
out.append((value, line) if raw else value)
return out
def fork_records(lines, upto=None):
"""[k, h] records of lines 2.. (those with seq <= `upto`), snapshots left
out: a fork encodes its copied snapshots again."""
out = []
for value, line in lines[1:]:
if upto is not None and value.get("seq", 0) > upto:
break
kind = value.get("kind")
if kind == "snapshot":
continue
out.append({"k": TURN_KINDS.get(kind, kind), "h": hashlib.sha256(line).hexdigest()[:16]})
return out
def fork_target(lines, root):
"""For a fork, its copied lines and its source's lines; else None."""
origin = lines[0][0].get("forked_from")
if not origin:
return None
source = os.path.join(root, origin["id"], "log.jsonl")
if not os.path.isfile(source):
raise Unreadable(f"fork source {origin['id']} is not in this root")
return fork_records(lines, origin["seq"]), fork_records(read_lines(source, raw=True))
def turn_target(lines):
target = []
for n, line in enumerate(lines, 1):
kind = line.get("kind")
if kind in OUTSIDE:
continue
if kind not in TURN_KINDS:
raise Unreadable(f"line {n} has unknown kind {kind!r}")
target.append(TURN_KINDS[kind])
return target
def child_target(lines, root):
names, works, target = {}, {}, []
for line in lines:
kind = line.get("kind")
if kind not in CHILD_KINDS:
continue
child = line["child"]
c = names.setdefault(child, f"c{len(names) + 1}")
if kind == "child_spawned":
works[c] = works.get(c, 0) + 1
target.append({"k": "spawned", "c": c, "w": works[c], "o": "none"})
else:
target.append({"k": "finished", "c": c, "w": works.get(c, 0), "o": line["outcome"]})
final = {c: os.path.exists(os.path.join(root, child, "log.jsonl")) for child, c in names.items()}
return target, final
def tla_str(s):
return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"'
def tla_record(r):
return "[" + ", ".join(f"{k} |-> {tla_str(v) if isinstance(v, str) else v}" for k, v in r.items()) + "]"
def run_tlc(module, spec_files, generated, cfg, invariant="NotWritten"):
work = tempfile.mkdtemp(prefix="fx-log-check-")
try:
for f in spec_files:
shutil.copy(f, work)
for name, text in generated.items():
open(os.path.join(work, name), "w").write(text)
open(os.path.join(work, module + ".cfg"), "w").write(cfg)
proc = subprocess.run(
# Its own java.io.tmpdir: TLC unpacks its standard modules there,
# and parallel runs sharing one folder read each other's
# half-written files.
[JAVA, "-XX:+UseParallelGC", f"-Djava.io.tmpdir={work}", "-cp", JAR, "tlc2.TLC", "-workers", "1", "-deadlock",
"-metadir", os.path.join(work, "states"), "-config", module + ".cfg", module],
cwd=work, capture_output=True, text=True, timeout=600,
)
out = proc.stdout + proc.stderr
matched = max([int(m) for m in re.findall(r'"matched_lines", (\d+)', out)] or [0])
if f"Invariant {invariant} is violated" in out:
return "ACCEPTED", matched
if "No error has been found" in out:
return "REJECTED", matched
return "TOOL ERROR", out[-600:]
finally:
shutil.rmtree(work, ignore_errors=True)
def check_turns(target):
body = ", ".join(tla_str(k) for k in target)
saved = f"---- MODULE SavedLog ----\nTarget == <<{body}>>\n====\n"
cfg = (f"CONSTANT MaxLen = {len(target) + 2}\nSPECIFICATION LogSpec\n"
"INVARIANT NotWritten\nCONSTRAINT Progress\n")
specs = [os.path.join(TLA, "TurnLifecycle.tla"), os.path.join(HERE, "TurnLifecycleLog.tla")]
return run_tlc("TurnLifecycleLog", specs, {"SavedLog.tla": saved}, cfg)
def check_children(target, final):
children = sorted(final)
body = ", ".join(tla_record(r) for r in target)
fin = " @@ ".join(f"{tla_str(c)} :> {'TRUE' if final[c] else 'FALSE'}" for c in children)
saved = (f"---- MODULE SavedChildren ----\nEXTENDS TLC\nTarget == <<{body}>>\n"
f"FinalChildLog == {fin}\n====\n")
max_work = max([r["w"] for r in target] or [1])
cfg = ("CONSTANTS Children = {" + ", ".join(tla_str(c) for c in children) + "}\n"
f" MaxWork = {max_work}\nSPECIFICATION LogSpec\n"
"INVARIANT NotWritten\nCONSTRAINT Progress\n")
specs = [os.path.join(TLA, "Subagents.tla"), os.path.join(HERE, "SubagentsLog.tla")]
return run_tlc("SubagentsLog", specs, {"SavedChildren.tla": saved}, cfg)
def check_fork(copied, source):
def seq(rs):
return ", ".join(tla_record(r) for r in rs)
saved = f"---- MODULE SavedFork ----\nCopied == <<{seq(copied)}>>\nSource == <<{seq(source)}>>\n====\n"
cfg = "SPECIFICATION LogSpec\nINVARIANT NotWritten\n"
verdict, _ = run_tlc("ForkLog", [os.path.join(HERE, "ForkLog.tla")], {"SavedFork.tla": saved}, cfg)
return verdict, len(copied)
def jobs_for(roots):
jobs = []
for root in roots:
for sid in sorted(os.listdir(root)):
path = os.path.join(root, sid, "log.jsonl")
if not os.path.isfile(path):
continue
name = f"{os.path.basename(os.path.dirname(root.rstrip('/')))}/{sid}"
try:
raw = read_lines(path, raw=True)
lines = [value for value, _ in raw]
turns = turn_target(lines)
children = child_target(lines, root)
fork = fork_target(raw, root)
except Unreadable as err:
jobs.append((name, "unreadable", str(err)))
continue
jobs.append((name, "turns", turns))
if children[0]:
jobs.append((name, "children", children))
if fork:
jobs.append((name, "fork", fork))
return jobs
def run(job):
name, what, arg = job
if what == "unreadable":
return name, what, "UNREADABLE", arg
if what == "turns":
verdict, detail = check_turns(arg)
elif what == "fork":
verdict, detail = check_fork(*arg)
else:
verdict, detail = check_children(*arg)
size = len(arg) if what == "turns" else len(arg[0])
return name, what, verdict, f"{detail}/{size} lines" if verdict != "TOOL ERROR" else detail
def selftest():
cases = [
("turns: a committed turn, then close", "turns", ["created", "set", "start", "item", "commit", "closed"], "ACCEPTED"),
("turns: crash, reopen interrupts, next turn", "turns", ["created", "start", "item", "interrupt", "start", "commit"], "ACCEPTED"),
("turns: close interrupts the open turn", "turns", ["created", "start", "interrupt", "closed", "start", "commit"], "ACCEPTED"),
("planted: item outside a turn", "turns", ["created", "item"], "REJECTED"),
("planted: two open turns", "turns", ["created", "start", "start"], "REJECTED"),
("planted: closed inside a turn", "turns", ["created", "start", "closed"], "REJECTED"),
("planted: commit without a start", "turns", ["created", "set", "commit"], "REJECTED"),
("children: spawn, finish ok",
"children", ([{"k": "spawned", "c": "c1", "w": 1, "o": "none"}, {"k": "finished", "c": "c1", "w": 1, "o": "ok"}], {"c1": True}), "ACCEPTED"),
("children: lost, then a fresh start (D33)",
"children", ([{"k": "spawned", "c": "c1", "w": 1, "o": "none"}, {"k": "finished", "c": "c1", "w": 1, "o": "lost"},
{"k": "spawned", "c": "c1", "w": 2, "o": "none"}, {"k": "finished", "c": "c1", "w": 2, "o": "ok"}], {"c1": True}), "ACCEPTED"),
("planted: lost, yet the child has a log",
"children", ([{"k": "spawned", "c": "c1", "w": 1, "o": "none"}, {"k": "finished", "c": "c1", "w": 1, "o": "lost"}], {"c1": True}), "REJECTED"),
("planted: interrupted, yet the child has no log",
"children", ([{"k": "spawned", "c": "c1", "w": 1, "o": "none"}, {"k": "finished", "c": "c1", "w": 1, "o": "interrupted"}], {"c1": False}), "REJECTED"),
("planted: a finish with no spawn",
"children", ([{"k": "finished", "c": "c1", "w": 0, "o": "ok"}], {"c1": False}), "REJECTED"),
]
def rec(k, h):
return {"k": k, "h": h}
src = [rec("set", "p"), rec("start", "a"), rec("item", "b"), rec("commit", "c"), rec("start", "d"), rec("commit", "e")]
cases += [
("fork: the source's lines to the end of turn 1", "fork", (src[:4], src), "ACCEPTED"),
("fork: before the first turn", "fork", (src[:1], src), "ACCEPTED"),
("planted: a copied line differs from the source's", "fork", (src[:2] + [rec("item", "x"), src[3]], src), "REJECTED"),
("planted: the copy ends inside a turn", "fork", (src[:3], src), "REJECTED"),
]
bad = 0
def verdict_of(c):
if c[1] == "turns":
return check_turns(c[2])
if c[1] == "fork":
return check_fork(*c[2])
return check_children(*c[2])
with ThreadPoolExecutor(int(os.environ.get("JOBS", "6"))) as pool:
results = pool.map(lambda c: (c, verdict_of(c)), cases)
for (label, _, _, want), (verdict, detail) in results:
ok = verdict == want
bad += not ok
print(f"{'ok ' if ok else 'FAIL'} {label}: {verdict} (matched {detail}), wanted {want}")
print(f"selftest: {len(cases) - bad}/{len(cases)} verdicts as required")
return 1 if bad else 0
def main(argv):
if argv[1:] == ["--selftest"]:
return selftest()
if len(argv) < 2:
print(__doc__)
return 2
jobs = jobs_for(argv[1:])
counts = {}
failed = 0
with ThreadPoolExecutor(int(os.environ.get("JOBS", "6"))) as pool:
for name, what, verdict, detail in pool.map(run, jobs):
counts[(what, verdict)] = counts.get((what, verdict), 0) + 1
if verdict not in ("ACCEPTED", "UNREADABLE"):
failed += 1
if verdict != "ACCEPTED":
print(f"{verdict:10} {what:9} {name}: {detail}")
for (what, verdict), n in sorted(counts.items()):
print(f"{what:9} {verdict:10} {n}")
return 1 if failed else 0
if __name__ == "__main__":
sys.exit(main(sys.argv))
@@ -0,0 +1,155 @@
#!/usr/bin/env python3
"""Checks fx's `wiring` trace lines against Wiring.tla (WiringTrace.tla).
usage: check_wiring_traces.py FILE... trace logs written with FX_TRACE=1,
FX_TRACE_SCOPES=wiring, FX_TRACE_LOG
check_wiring_traces.py --selftest
JOBS=N runs N TLC checks at once (default 6).
The adapter writes one line per step (`action=... session=... pid=...`).
Each process's first store open and last close become Start and Exit; a
process whose lines stop without Exit gets a Crash. Wiring.tla models one fx
process at a time, so a file whose processes overlap is skipped and named.
Exit 0 when every checked trace is accepted.
"""
import os
import re
import sys
from concurrent.futures import ThreadPoolExecutor
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from check_saved_logs import HERE, TLA, run_tlc, tla_str # noqa: E402
LINE = re.compile(r"^\d+ \[wiring\] action=(\w+) session=(\S+)(?: end=(\w+))? pid=(-?\d+)$")
class Skip(Exception):
pass
def parse(path):
events = []
for raw in open(path, errors="replace"):
m = LINE.match(raw.rstrip("\n"))
if m:
events.append((int(m.group(4)), m.group(1), m.group(2), m.group(3)))
return events
def steps_for(events):
"""The Wiring steps, and the sessions they name."""
order, seen = [], set()
for pid, *_ in events:
if pid in seen and order[-1] != pid:
raise Skip("processes overlap")
if pid not in seen:
seen.add(pid)
order.append(pid)
names, steps = {}, []
for pid in order:
depth = 0
mine = [e for e in events if e[0] == pid]
# ACP opens its store before tracing starts: its first line is
# then a later step, and the process started just before it.
if mine[0][1] != "StoreOpen":
steps.append(("Start", "none", "none"))
depth = 1
for p, action, session, end in mine:
if action == "StoreOpen":
if depth == 0:
steps.append(("Start", "none", "none"))
depth += 1
elif action == "StoreClose":
depth -= 1
if depth == 0:
steps.append(("Exit", "none", "none"))
elif depth == 0:
raise Skip(f"{action} outside an open store")
elif action in ("Open", "Close", "Fork"):
steps.append((action, names.setdefault(session, f"s{len(names) + 1}"), "none"))
elif action == "EndTurn":
steps.append(("EndTurn", "none", end))
else:
steps.append((action, "none", "none"))
if depth > 0:
steps.append(("Crash", "none", "none"))
return steps, sorted(names.values()) or ["s1"]
def check(steps, sessions):
body = ", ".join(f"[a |-> {tla_str(a)}, s |-> {tla_str(s)}, e |-> {tla_str(e)}]" for a, s, e in steps)
data = f"---- MODULE WiringData ----\nTrace == <<{body}>>\n====\n"
cfg = ("CONSTANT Sessions = {" + ", ".join(tla_str(s) for s in sessions) + "}\n"
"SPECIFICATION TraceSpec\nINVARIANT NotDone\nCONSTRAINT Progress\n")
specs = [os.path.join(TLA, "Wiring.tla"), os.path.join(HERE, "WiringTrace.tla")]
verdict, detail = run_tlc("WiringTrace", specs, {"WiringData.tla": data}, cfg, invariant="NotDone")
return verdict, detail
def run(path):
events = parse(path)
if not events:
return path, "EMPTY", "no wiring lines"
try:
steps, sessions = steps_for(events)
except Skip as why:
return path, "SKIPPED", str(why)
verdict, detail = check(steps, sessions)
if verdict == "REJECTED":
step = steps[detail] if detail < len(steps) else None
return path, verdict, f"steps 1..{detail} match; step {detail + 1} {step} is not allowed ({len(steps)} steps)"
return path, verdict, f"{detail}/{len(steps)} steps" if verdict == "ACCEPTED" else detail
def selftest():
S, O, B, C, X = ("Start", "none", "none"), ("Open", "s1", "none"), ("BeginTurn", "none", "none"), ("Close", "s1", "none"), ("Exit", "none", "none")
O2, C2 = ("Open", "s2", "none"), ("Close", "s2", "none")
commit, crash = ("EndTurn", "none", "commit"), ("Crash", "none", "none")
spawn, done = ("Spawn", "none", "none"), ("ChildDone", "none", "none")
cases = [
("a turn, then close and exit", [S, O, B, commit, C, X], ["s1"], "ACCEPTED"),
("a crash mid-turn, then a resume", [S, O, B, crash, S, O, B, commit, C, X], ["s1"], "ACCEPTED"),
("a child inside a turn", [S, O, B, spawn, done, commit, C, X], ["s1"], "ACCEPTED"),
("planted: close while a child runs", [S, O, B, spawn, C], ["s1"], "REJECTED"),
("a switch: the next session opens before the current closes (D41)", [S, O, B, commit, O2, C, B, commit, C2, X], ["s1", "s2"], "ACCEPTED"),
("a failed switch closes the next session and keeps the current", [S, O, O2, C2, B, commit, C, X], ["s1", "s2"], "ACCEPTED"),
("planted: a third open while a switch is pending", [S, O, O2, ("Open", "s3", "none")], ["s1", "s2", "s3"], "REJECTED"),
("planted: a turn starts while a switch is pending", [S, O, O2, B], ["s1", "s2"], "REJECTED"),
("planted: a turn ends that never began", [S, O, commit], ["s1"], "REJECTED"),
("planted: exit with a session open", [S, O, X], ["s1"], "REJECTED"),
("planted: a child done after its parent closed", [S, O, B, spawn, commit, C, done], ["s1"], "REJECTED"),
("a fork becomes the next session, and closing the source switches to it (D58)", [S, O, B, commit, ("Fork", "s2", "none"), C, B, commit, C2, X], ["s1", "s2"], "ACCEPTED"),
("planted: a fork while a turn is open", [S, O, B, ("Fork", "s2", "none")], ["s1", "s2"], "REJECTED"),
("planted: a fork into a session that exists", [S, O2, C2, O, ("Fork", "s2", "none")], ["s1", "s2"], "REJECTED"),
]
bad = 0
with ThreadPoolExecutor(int(os.environ.get("JOBS", "6"))) as pool:
for (label, _, _, want), (verdict, detail) in zip(cases, pool.map(lambda c: check(c[1], c[2]), cases)):
ok = verdict == want
bad += not ok
print(f"{'ok ' if ok else 'FAIL'} {label}: {verdict} (matched {detail}), wanted {want}")
print(f"selftest: {len(cases) - bad}/{len(cases)} verdicts as required")
return 1 if bad else 0
def main(argv):
if argv[1:] == ["--selftest"]:
return selftest()
if len(argv) < 2:
print(__doc__)
return 2
counts, failed = {}, 0
with ThreadPoolExecutor(int(os.environ.get("JOBS", "6"))) as pool:
for path, verdict, detail in pool.map(run, argv[1:]):
counts[verdict] = counts.get(verdict, 0) + 1
failed += verdict in ("REJECTED", "TOOL ERROR")
if verdict != "ACCEPTED":
print(f"{verdict:10} {path}: {detail}")
for verdict, n in sorted(counts.items()):
print(f"{verdict:10} {n}")
return 1 if failed else 0
if __name__ == "__main__":
sys.exit(main(sys.argv))
+42
View File
@@ -0,0 +1,42 @@
#!/bin/sh
# Prove the trace tooling can both accept and reject.
#
# Runs the Python stand-in log three ways and checks the verdicts:
# correct implementation -> ACCEPTED
# recovery keeps a torn tail (bug) -> REJECTED at step 8 (the Recover)
# one field changed in a good trace -> REJECTED at step 5 (acked 2 -> 3)
set -eu
here=$(cd "$(dirname "$0")" && pwd)
tmp=$(mktemp -d "${TMPDIR:-/tmp}/session-manager-selftest.XXXXXX")
trap 'rm -rf "$tmp"' EXIT
python3 "$here/stand_in_log.py" "$tmp/good.log" "$tmp/good.ndjson"
python3 "$here/stand_in_log.py" "$tmp/bad.log" "$tmp/bad.ndjson" keep_torn_tail
sed '5s/"acked": 2/"acked": 3/' "$tmp/good.ndjson" > "$tmp/tampered.ndjson"
failures=0
expect() { # name trace wanted_exit wanted_text
set +e
got=$("$here/validate.sh" SessionLog "$2" 2>&1)
code=$?
set -e
if [ "$code" -eq "$3" ] && printf '%s' "$got" | grep -q "$4"; then
echo "ok $1: $(printf '%s' "$got" | head -1)"
else
echo "FAIL $1: exit $code, wanted $3 and '$4'"
printf '%s\n' "$got" | sed 's/^/ /'
failures=$((failures + 1))
fi
}
expect "correct implementation" "$tmp/good.ndjson" 0 "ACCEPTED"
expect "recovery keeps a torn tail" "$tmp/bad.ndjson" 1 "step 8 of"
expect "tampered acked field" "$tmp/tampered.ndjson" 1 "step 5 of"
if [ "$failures" -eq 0 ]; then
echo "trace selftest passed"
exit 0
fi
echo "trace selftest failed: $failures case(s)"
exit 1
+115
View File
@@ -0,0 +1,115 @@
#!/usr/bin/env python3
"""A small stand-in for the L1 log, used only to self-test the trace tooling.
It writes a real file with O_APPEND, fsync and simulated crashes, and emits
one trace line per step. The state in each line is read back from the actual
file bytes, the same way the Zig trace hook must do it. Not a design
reference for the Zig code.
usage: stand_in_log.py LOG_PATH TRACE_PATH [keep_torn_tail]
"""
import json
import os
import sys
class Log:
def __init__(self, path, trace, bug=None):
self.path, self.trace, self.bug = path, trace, bug
self.fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
self.durable = self.acked = 0
self.status, self.pending, self.seq = "open", None, 0
def disk(self):
"""Abstraction function: file bytes -> (line count, tail done, seqs)."""
with open(self.path, "rb") as f:
data = f.read()
torn = len(data) > 0 and not data.endswith(b"\n")
complete = data.split(b"\n")[: data.count(b"\n")]
seqs = []
for raw in complete:
try:
seqs.append(int(json.loads(raw)["seq"]))
except (ValueError, KeyError, TypeError):
seqs.append(-1)
return len(complete) + (1 if torn else 0), not torn, seqs
def emit(self, event):
n, done, seqs = self.disk()
self.trace.write(json.dumps({
"event": event, "log_len": n, "tail_done": done, "seqs": seqs,
"durable": self.durable, "acked": self.acked, "status": self.status,
}) + "\n")
def begin_write(self):
"""First half of one write(): the newline is not on disk yet."""
self.seq += 1
line = json.dumps({"v": 1, "seq": self.seq, "kind": "item"}).encode() + b"\n"
half = len(line) // 2
os.write(self.fd, line[:half])
self.pending = line[half:]
self.status = "writing"
self.emit("BeginWrite")
def finish_write(self):
os.write(self.fd, self.pending)
self.pending = None
self.status = "open"
self.emit("FinishWrite")
def fsync(self):
os.fsync(self.fd)
self.durable = self.acked = self.disk()[0]
self.emit("Fsync")
def process_crash(self):
os.close(self.fd)
self.status = "down"
self.emit("ProcessCrash")
def power_loss(self, keep, tear):
"""Simulated: keep `keep` complete lines (at least durable), then tear
the last kept line if it was never fsynced."""
if self.status != "down":
os.close(self.fd)
with open(self.path, "rb") as f:
data = f.read()
kept = b"".join(p + b"\n" for p in data.split(b"\n")[:keep])
if tear and keep > self.durable:
kept = kept[:-3]
with open(self.path, "wb") as f:
f.write(kept)
self.status = "down"
self.emit("PowerLoss")
def recover(self):
"""open(.resume): cut a torn tail, fsync, reopen for append."""
with open(self.path, "rb") as f:
data = f.read()
if data and not data.endswith(b"\n") and self.bug != "keep_torn_tail":
data = data[: data.rfind(b"\n") + 1]
with open(self.path, "wb") as f:
f.write(data)
self.fd = os.open(self.path, os.O_WRONLY | os.O_APPEND)
os.fsync(self.fd)
self.durable = self.disk()[0]
self.seq = data.count(b"\n")
self.status = "open"
self.emit("Recover")
def scenario(path, trace_path, bug=None):
with open(trace_path, "w") as trace:
log = Log(path, trace, bug)
log.begin_write(); log.finish_write()
log.begin_write(); log.finish_write(); log.fsync() # 2 acknowledged lines
log.begin_write(); log.process_crash() # torn 3rd line
log.recover() # must cut it
log.begin_write(); log.finish_write() # new 3rd line, not fsynced
log.power_loss(keep=3, tear=True) # machine crash tears it
log.recover()
log.begin_write(); log.finish_write(); log.fsync()
if __name__ == "__main__":
scenario(sys.argv[1], sys.argv[2], sys.argv[3] if len(sys.argv) > 3 else None)
+59
View File
@@ -0,0 +1,59 @@
#!/usr/bin/env python3
"""Convert an implementation trace (JSON lines) into the TLA+ module TraceData.
usage: to_tla.py trace.ndjson TraceData.tla
Each JSON line becomes one record in `Trace == << ... >>`. Values map as:
true/false -> TRUE/FALSE, integers -> integers, strings -> strings,
lists -> sequences, objects -> records, {"$set": [...]} -> a set.
Floats and null are rejected: the specs have no use for them.
"""
import json
import sys
def value(v):
if isinstance(v, bool):
return "TRUE" if v else "FALSE"
if isinstance(v, int):
return str(v)
if isinstance(v, str):
return json.dumps(v)
if isinstance(v, list):
return "<<" + ", ".join(value(x) for x in v) + ">>"
if isinstance(v, dict):
if set(v) == {"$set"}:
return "{" + ", ".join(value(x) for x in v["$set"]) + "}"
if not v:
raise ValueError("empty objects have no TLA+ record form")
return "[" + ", ".join(f"{k} |-> {value(x)}" for k, x in v.items()) + "]"
raise ValueError(f"unsupported trace value: {v!r}")
def main():
if len(sys.argv) != 3:
print(__doc__.strip().splitlines()[2], file=sys.stderr)
return 2
records = []
with open(sys.argv[1]) as trace:
for number, line in enumerate(trace, start=1):
if not line.strip():
continue
try:
records.append(value(json.loads(line)))
except (ValueError, json.JSONDecodeError) as err:
print(f"trace line {number}: {err}", file=sys.stderr)
return 2
body = ",\n ".join(records)
with open(sys.argv[2], "w") as out:
out.write("---- MODULE TraceData ----\n")
# Integers gives negative literals (for example a seq of -1 for a
# line the implementation could not parse).
out.write("EXTENDS Integers\n")
out.write(f"Trace == <<\n {body}\n>>\n" if records else "Trace == <<>>\n")
out.write("====\n")
return 0
if __name__ == "__main__":
sys.exit(main())
+79
View File
@@ -0,0 +1,79 @@
#!/bin/sh
# Check that an implementation trace is a behavior of a spec.
#
# usage: ./validate.sh SPEC trace.ndjson
# SPEC is a spec with a trace wrapper here, for example SessionLog.
# Needs ../SPEC.tla, ./SPECTrace.tla and ./SPECTrace.cfg.
#
# Exit codes: 0 accepted, 1 rejected (prints the first step the spec does
# not allow), 2 tool or usage error.
set -u
if [ "$#" -ne 2 ]; then
echo "usage: $0 SPEC trace.ndjson" >&2
exit 2
fi
spec=$1
here=$(cd "$(dirname "$0")" && pwd)
trace=$(cd "$(dirname "$2")" && pwd)/$(basename "$2")
for f in "$here/../$spec.tla" "$here/${spec}Trace.tla" "$here/${spec}Trace.cfg" "$trace"; do
if [ ! -f "$f" ]; then
echo "missing $f" >&2
exit 2
fi
done
jar="${TLA2TOOLS_JAR:-$HOME/.local/share/tla/tla2tools.jar}"
java=""
for candidate in \
"${JAVA_HOME:+$JAVA_HOME/bin/java}" \
/opt/homebrew/opt/openjdk/bin/java \
/usr/local/opt/openjdk/bin/java \
"$(command -v java 2>/dev/null || true)"; do
if [ -n "$candidate" ] && [ -x "$candidate" ] && "$candidate" -version >/dev/null 2>&1; then
java="$candidate"
break
fi
done
if [ -z "$java" ] || [ ! -f "$jar" ]; then
echo "needs a Java runtime and tla2tools.jar (see ../check.sh)" >&2
exit 2
fi
work=$(mktemp -d "${TMPDIR:-/tmp}/session-manager-trace.XXXXXX")
trap 'rm -rf "$work"' EXIT
cp "$here/../$spec.tla" "$here/${spec}Trace.tla" "$here/${spec}Trace.cfg" "$work/"
python3 "$here/to_tla.py" "$trace" "$work/TraceData.tla" || exit 2
steps=$(grep -c . "$trace" || true)
# One worker: the progress register (TLCSet/TLCGet) is per worker. TLC
# unpacks its standard modules into java.io.tmpdir; a private one lets
# several checks run at once.
mkdir "$work/java-tmp"
out=$(cd "$work" && "$java" -Djava.io.tmpdir="$work/java-tmp" -cp "$jar" tlc2.TLC -deadlock -workers 1 -cleanup \
-metadir "$work/states" -config "${spec}Trace.cfg" "${spec}Trace.tla" 2>&1)
# Match with case, not a pipe into grep -q: grep exits at the first match
# and a large TLC output then fails printf with a broken pipe.
case $out in
*'Invariant TraceIncomplete is violated'*) verdict=accepted ;;
*'No error has been found'*) verdict=rejected ;;
*) verdict=unfinished ;;
esac
if [ "$verdict" = accepted ]; then
echo "ACCEPTED: all $steps steps are a behavior of $spec"
exit 0
fi
if [ "$verdict" = rejected ]; then
matched=$(printf '%s\n' "$out" | sed -n 's/.*"matched_lines", \([0-9]*\)>>.*/\1/p' | tail -1)
matched=${matched:-0}
bad=$((matched + 1))
echo "REJECTED: steps 1..$matched match; step $bad of $steps is not allowed by $spec:"
sed -n "${bad}p" "$trace"
exit 1
fi
printf '%s\n' "$out" | tail -15
echo "TOOL ERROR: TLC did not finish the trace check" >&2
exit 2
+1 -1
View File
@@ -2,7 +2,7 @@
//!
//! Each traced run writes `zig-out/traces/SPEC--CASE.ndjson`: one JSON line
//! per spec action, named like the action, with one field per spec variable
//! (`docs/session-manager/tla/trace/README.md`). Fields that model the disk
//! (`tla/trace/README.md`). Fields that model the disk
//! are read back from the real files after the step; fields that model
//! memory come from the code's own state.
//!
+1 -1
View File
@@ -153,7 +153,7 @@ pub const Runtime = struct {
/// A host over a v2 parent, whose children live in its log (D22).
/// Borrows `children` and the callback contexts until deinit; the
/// parent stays open until then (`tla/Wiring.tla`
/// parent stays open until then (`src/core/session_manager/tla/Wiring.tla`
/// ParentOutlivesChildren).
pub fn createV2(
alloc: Allocator,
+1 -1
View File
@@ -1,4 +1,4 @@
//! JSONL trace of core steps (docs/approach.md §6, format version 1).
//! JSONL trace of core steps (format version 1).
//!
//! The I/O layer writes one line per `step` call when tracing is on. The Bun
//! bridge turns a trace into a TLA+ observation and asks TLC whether it is a
+1 -1
View File
@@ -1,7 +1,7 @@
//! MCPv2 module root and public API.
//!
//! Files under `src/mcp/` import only `std` and each other. The host passes in
//! `std.Io` and everything else through `host.zig`. See `docs/approach.md`.
//! `std.Io` and everything else through `host.zig`.
const std = @import("std");
+3 -3
View File
@@ -2025,9 +2025,9 @@ test("a crash while a named subagent works records it interrupted, and its next
expect(JSON.parse(resumed.stdout).output).toBe("AFTER_LOST_DONE");
// The reopen recorded the started work as interrupted, as
// `tla/Subagents.tla` repairs a child with a log; the next message is new
// work for the same child. A child streams no pieces, so its log holds
// nothing of the lost work.
// `src/core/session_manager/tla/Subagents.tla` repairs a child with a
// log; the next message is new work for the same child. A child streams
// no pieces, so its log holds nothing of the lost work.
const lines = childLines(fixture, id);
expect(lines.map((line) => [line.kind, line.outcome ?? null])).toEqual([
["child_spawned", null], ["child_finished", "interrupted"], ["child_spawned", null], ["child_finished", "ok"],