{ "version": "2.0", "generated_by": "rank.md 2.0 (scripts/build-vectors.mts from src/lib/rank/vectors.ts)", "about": "RANK.md format version 1 test vectors: inputs and the exact outputs the rank.md code returns. A conforming implementation returns the same outputs. Rebuild with npm run vectors. Formulas: https://rank.md/methods.md. Spec: https://rank.md/spec.md. Example data only.", "vectors": [ { "name": "sha256Hex of \"\"", "fn": "sha256Hex", "input": "", "output": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "name": "sha256Hex of \"rank\"", "fn": "sha256Hex", "input": "rank", "output": "5a8460f6b1bf96a87116e44e3dc74ef825d068d893516128e0fcbe4e646bc06d" }, { "name": "sha256Hex of \"RANK.md format version 1\"", "fn": "sha256Hex", "input": "RANK.md format version 1", "output": "91fddebd8d7cf02b206b8413f08bcd7e8967750e4516de9af0398927e0f94da8" }, { "name": "canonicalJson sorts keys and drops undefined", "fn": "canonicalJson", "input": { "b": [ 2, 1 ], "a": { "d": null, "c": "x" } }, "output": "{\"a\":{\"c\":\"x\",\"d\":null},\"b\":[2,1]}" }, { "name": "canonicalJson of numbers", "fn": "canonicalJson", "input": [ 0.1, 1e+21, 0, 2.5e-7 ], "output": "[0.1,1e+21,0,2.5e-7]" }, { "name": "normalizeText", "fn": "normalizeText", "input": " Layout\tPARSER ", "output": "layout parser" }, { "name": "normalizeUri drops the fragment and trailing slash", "fn": "normalizeUri", "input": "HTTPS://Example.COM:443/Tools/Parser/#top", "output": "https://example.com/Tools/Parser" }, { "name": "itemId of {\"label\":\"Layout parser\"}", "fn": "itemId", "input": { "label": "Layout parser" }, "output": "it_3c2349db14c0d212" }, { "name": "itemId of {\"label\":\" layout PARSER \"}", "fn": "itemId", "input": { "label": " layout PARSER " }, "output": "it_3c2349db14c0d212" }, { "name": "itemId of {\"label\":\"Layout parser\",\"uri\":\"https://example.com/parser/\"}", "fn": "itemId", "input": { "label": "Layout parser", "uri": "https://example.com/parser/" }, "output": "it_dc3ce730378846ec" }, { "name": "itemId of {\"label\":\"Layout parser\",\"key\":\"parser:layout\"}", "fn": "itemId", "input": { "label": "Layout parser", "key": "parser:layout" }, "output": "it_195c745ceac8f408" }, { "name": "rankingId of a tool question", "fn": "rankingId", "input": { "subject": "tool", "question": "Which tool should my agent call for this task?" }, "output": "rk_d9b7da8869386f57" }, { "name": "rankingId ignores case and spacing", "fn": "rankingId", "input": { "subject": "tool", "question": " which TOOL should my agent call for this task?" }, "output": "rk_d9b7da8869386f57" }, { "name": "snapshotId rounds scores to 6 decimals", "fn": "snapshotId", "input": { "as_of": "2026-10-24T18:00:00Z", "method": "weighted_sum", "weights": { "accuracy": 0.6, "latency_ms": 0.4 }, "items": [ { "id": "it_aaaaaaaaaaaaaaaa", "rank": 1, "score": 0.8123456789 }, { "id": "it_bbbbbbbbbbbbbbbb", "rank": 2, "score": 0.5 } ] }, "output": "rs_28912f8d2f7f92f5" }, { "name": "changeId", "fn": "changeId", "input": { "at": "2026-10-24T18:00:00Z", "type": "overtook", "item": "it_aaaaaaaaaaaaaaaa", "other": "it_bbbbbbbbbbbbbbbb", "from": 2, "to": 1 }, "output": "rc_686b5b755a2a353c" }, { "name": "parseDuration \"90s\"", "fn": "parseDuration", "input": "90s", "output": 90000 }, { "name": "parseDuration \"1h 30m\"", "fn": "parseDuration", "input": "1h 30m", "output": 5400000 }, { "name": "parseDuration \"PT6H\"", "fn": "parseDuration", "input": "PT6H", "output": 21600000 }, { "name": "parseDuration \"2w\"", "fn": "parseDuration", "input": "2w", "output": 1209600000 }, { "name": "parseDuration \"6 hours\"", "fn": "parseDuration", "input": "6 hours", "output": null }, { "name": "parseInstant \"2026-10-24T18:00:00Z\"", "fn": "parseInstant", "input": "2026-10-24T18:00:00Z", "output": 1792864800000 }, { "name": "parseInstant \"2026-10-24T11:00:00-07:00\"", "fn": "parseInstant", "input": "2026-10-24T11:00:00-07:00", "output": 1792864800000 }, { "name": "parseInstant \"2026-10-24\"", "fn": "parseInstant", "input": "2026-10-24", "output": 1792800000000 }, { "name": "parseInstant \"2026-02-30T00:00:00Z\"", "fn": "parseInstant", "input": "2026-02-30T00:00:00Z", "output": null }, { "name": "formatDuration", "fn": "formatDuration", "input": 7800000, "output": "2h 10m" }, { "name": "rankItems method weighted_sum", "fn": "rankItems", "input": { "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio" }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 } }, { "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 } }, { "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 } }, { "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": 40, "cost": 0.001 } } ], "method": "weighted_sum" }, "output": { "weights": { "accuracy": 0.5, "latency_ms": 0.3, "cost": 0.2 }, "items": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "rank": 1, "score": 0.868798, "normalized": { "accuracy": 1, "latency_ms": 0.815534, "cost": 0.62069 }, "why": "accuracy 0.94 (+0.50), latency_ms 420 (+0.24)", "move": null }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "rank": 2, "score": 0.83353, "normalized": { "accuracy": 0.818182, "latency_ms": 0.932039, "cost": 0.724138 }, "why": "accuracy 0.9 (+0.41), latency_ms 180 (+0.28)", "move": null }, { "id": "it_e7567efc82a32053", "label": "Rules engine", "rank": 3, "score": 0.5, "normalized": { "accuracy": 0, "latency_ms": 1, "cost": 1 }, "why": "latency_ms 40 (+0.30), cost 0.001 (+0.20)", "move": null }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "rank": 4, "score": 0.454545, "normalized": { "accuracy": 0.909091, "latency_ms": 0, "cost": 0 }, "why": "accuracy 0.92 (+0.45), latency_ms 2100 (+0.00)", "move": null } ], "excluded": [], "snapshot": "rs_3e7581b14bf9dc45", "notes": [] } }, { "name": "rankItems method weighted_product", "fn": "rankItems", "input": { "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio" }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 } }, { "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 } }, { "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 } }, { "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": 40, "cost": 0.001 } } ], "method": "weighted_product" }, "output": { "weights": { "accuracy": 0.5, "latency_ms": 0.3, "cost": 0.2 }, "items": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "rank": 1, "score": 0.855082, "normalized": { "accuracy": 1, "latency_ms": 0.815534, "cost": 0.62069 }, "why": "accuracy 0.94 (+0.50), latency_ms 420 (+0.24)", "move": null }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "rank": 2, "score": 0.83027, "normalized": { "accuracy": 0.818182, "latency_ms": 0.932039, "cost": 0.724138 }, "why": "accuracy 0.9 (+0.41), latency_ms 180 (+0.28)", "move": null }, { "id": "it_e7567efc82a32053", "label": "Rules engine", "rank": 3, "score": 0.1, "normalized": { "accuracy": 0, "latency_ms": 1, "cost": 1 }, "why": "latency_ms 40 (+0.30), cost 0.001 (+0.20)", "move": null }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "rank": 4, "score": 0.095346, "normalized": { "accuracy": 0.909091, "latency_ms": 0, "cost": 0 }, "why": "accuracy 0.92 (+0.45), latency_ms 2100 (+0.00)", "move": null } ], "excluded": [], "snapshot": "rs_3a2742d1e70a4320", "notes": [] } }, { "name": "rankItems method topsis", "fn": "rankItems", "input": { "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio" }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 } }, { "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 } }, { "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 } }, { "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": 40, "cost": 0.001 } } ], "method": "topsis" }, "output": { "weights": { "accuracy": 0.5, "latency_ms": 0.3, "cost": 0.2 }, "items": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "rank": 1, "score": 0.85863, "normalized": { "accuracy": 1, "latency_ms": 0.815534, "cost": 0.62069 }, "why": "accuracy 0.94 (+0.50), latency_ms 420 (+0.24)", "move": null }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "rank": 2, "score": 0.826624, "normalized": { "accuracy": 0.818182, "latency_ms": 0.932039, "cost": 0.724138 }, "why": "accuracy 0.9 (+0.41), latency_ms 180 (+0.28)", "move": null }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "rank": 3, "score": 0.55571, "normalized": { "accuracy": 0.909091, "latency_ms": 0, "cost": 0 }, "why": "accuracy 0.92 (+0.45), latency_ms 2100 (+0.00)", "move": null }, { "id": "it_e7567efc82a32053", "label": "Rules engine", "rank": 4, "score": 0.41898, "normalized": { "accuracy": 0, "latency_ms": 1, "cost": 1 }, "why": "latency_ms 40 (+0.30), cost 0.001 (+0.20)", "move": null } ], "excluded": [], "snapshot": "rs_e336018ad3989519", "notes": [] } }, { "name": "rankItems method lexicographic", "fn": "rankItems", "input": { "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio" }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 } }, { "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 } }, { "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 } }, { "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": 40, "cost": 0.001 } } ], "method": "lexicographic" }, "output": { "weights": { "accuracy": 0.5, "latency_ms": 0.3, "cost": 0.2 }, "items": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "rank": 1, "score": 0.868798, "normalized": { "accuracy": 1, "latency_ms": 0.815534, "cost": 0.62069 }, "why": "accuracy 0.94 (+0.50), latency_ms 420 (+0.24)", "move": null }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "rank": 2, "score": 0.454545, "normalized": { "accuracy": 0.909091, "latency_ms": 0, "cost": 0 }, "why": "accuracy 0.92 (+0.45), latency_ms 2100 (+0.00)", "move": null }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "rank": 3, "score": 0.83353, "normalized": { "accuracy": 0.818182, "latency_ms": 0.932039, "cost": 0.724138 }, "why": "accuracy 0.9 (+0.41), latency_ms 180 (+0.28)", "move": null }, { "id": "it_e7567efc82a32053", "label": "Rules engine", "rank": 4, "score": 0.5, "normalized": { "accuracy": 0, "latency_ms": 1, "cost": 1 }, "why": "latency_ms 40 (+0.30), cost 0.001 (+0.20)", "move": null } ], "excluded": [], "snapshot": "rs_6656ced23a6fbbd0", "notes": [] } }, { "name": "rankItems normalize minmax", "fn": "rankItems", "input": { "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio" }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 } }, { "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 } }, { "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 } }, { "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": 40, "cost": 0.001 } } ], "normalize": "minmax" }, "output": { "weights": { "accuracy": 0.5, "latency_ms": 0.3, "cost": 0.2 }, "items": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "rank": 1, "score": 0.868798, "normalized": { "accuracy": 1, "latency_ms": 0.815534, "cost": 0.62069 }, "why": "accuracy 0.94 (+0.50), latency_ms 420 (+0.24)", "move": null }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "rank": 2, "score": 0.83353, "normalized": { "accuracy": 0.818182, "latency_ms": 0.932039, "cost": 0.724138 }, "why": "accuracy 0.9 (+0.41), latency_ms 180 (+0.28)", "move": null }, { "id": "it_e7567efc82a32053", "label": "Rules engine", "rank": 3, "score": 0.5, "normalized": { "accuracy": 0, "latency_ms": 1, "cost": 1 }, "why": "latency_ms 40 (+0.30), cost 0.001 (+0.20)", "move": null }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "rank": 4, "score": 0.454545, "normalized": { "accuracy": 0.909091, "latency_ms": 0, "cost": 0 }, "why": "accuracy 0.92 (+0.45), latency_ms 2100 (+0.00)", "move": null } ], "excluded": [], "snapshot": "rs_3e7581b14bf9dc45", "notes": [] } }, { "name": "rankItems normalize rank", "fn": "rankItems", "input": { "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio" }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 } }, { "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 } }, { "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 } }, { "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": 40, "cost": 0.001 } } ], "normalize": "rank" }, "output": { "weights": { "accuracy": 0.5, "latency_ms": 0.3, "cost": 0.2 }, "items": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "rank": 1, "score": 0.666667, "normalized": { "accuracy": 1, "latency_ms": 0.333333, "cost": 0.333333 }, "why": "accuracy 0.94 (+0.50), latency_ms 420 (+0.10)", "move": null }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "rank": 2, "score": 0.5, "normalized": { "accuracy": 0.333333, "latency_ms": 0.666667, "cost": 0.666667 }, "why": "latency_ms 180 (+0.20), accuracy 0.9 (+0.17)", "move": null }, { "id": "it_e7567efc82a32053", "label": "Rules engine", "rank": 2, "score": 0.5, "normalized": { "accuracy": 0, "latency_ms": 1, "cost": 1 }, "why": "latency_ms 40 (+0.30), cost 0.001 (+0.20)", "move": null }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "rank": 4, "score": 0.333333, "normalized": { "accuracy": 0.666667, "latency_ms": 0, "cost": 0 }, "why": "accuracy 0.92 (+0.33), latency_ms 2100 (+0.00)", "move": null } ], "excluded": [], "snapshot": "rs_02e503cf3c706156", "notes": [] } }, { "name": "rankItems normalize zscore", "fn": "rankItems", "input": { "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio" }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 } }, { "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 } }, { "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 } }, { "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": 40, "cost": 0.001 } } ], "normalize": "zscore" }, "output": { "weights": { "accuracy": 0.5, "latency_ms": 0.3, "cost": 0.2 }, "items": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "rank": 1, "score": 0.688906, "normalized": { "accuracy": 0.787485, "latency_ms": 0.625508, "cost": 0.537557 }, "why": "accuracy 0.94 (+0.39), latency_ms 420 (+0.19)", "move": null }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "rank": 2, "score": 0.66498, "normalized": { "accuracy": 0.63378, "latency_ms": 0.728994, "cost": 0.646959 }, "why": "accuracy 0.9 (+0.32), latency_ms 180 (+0.22)", "move": null }, { "id": "it_e7567efc82a32053", "label": "Rules engine", "rank": 3, "score": 0.430641, "normalized": { "accuracy": 0.043688, "latency_ms": 0.781957, "cost": 0.87105 }, "why": "latency_ms 40 (+0.23), cost 0.001 (+0.17)", "move": null }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "rank": 4, "score": 0.381825, "normalized": { "accuracy": 0.715594, "latency_ms": 0.043765, "cost": 0.054492 }, "why": "accuracy 0.92 (+0.36), latency_ms 2100 (+0.01)", "move": null } ], "excluded": [], "snapshot": "rs_83010277039ff0ce", "notes": [] } }, { "name": "rankItems normalize log", "fn": "rankItems", "input": { "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio" }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 } }, { "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 } }, { "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 } }, { "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": 40, "cost": 0.001 } } ], "normalize": "log" }, "output": { "weights": { "accuracy": 0.5, "latency_ms": 0.3, "cost": 0.2 }, "items": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "rank": 1, "score": 0.745971, "normalized": { "accuracy": 1, "latency_ms": 0.408357, "cost": 0.61732 }, "why": "accuracy 0.94 (+0.50), cost 0.012 (+0.12)", "move": null }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "rank": 2, "score": 0.744545, "normalized": { "accuracy": 0.826907, "latency_ms": 0.62279, "cost": 0.721273 }, "why": "accuracy 0.9 (+0.41), latency_ms 180 (+0.19)", "move": null }, { "id": "it_e7567efc82a32053", "label": "Rules engine", "rank": 3, "score": 0.5, "normalized": { "accuracy": 0, "latency_ms": 1, "cost": 1 }, "why": "latency_ms 40 (+0.30), cost 0.001 (+0.20)", "move": null }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "rank": 4, "score": 0.456952, "normalized": { "accuracy": 0.913904, "latency_ms": 0, "cost": 0 }, "why": "accuracy 0.92 (+0.46), latency_ms 2100 (+0.00)", "move": null } ], "excluded": [], "snapshot": "rs_d2e02b767ecdbae2", "notes": [] } }, { "name": "rankItems normalize none", "fn": "rankItems", "input": { "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio" }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 } }, { "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 } }, { "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 } }, { "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": 40, "cost": 0.001 } } ], "normalize": "none" }, "output": { "weights": { "accuracy": 0.5, "latency_ms": 0.3, "cost": 0.2 }, "items": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "rank": 1, "score": 0.6676, "normalized": { "accuracy": 0.94, "latency_ms": 0, "cost": 0.988 }, "why": "accuracy 0.94 (+0.47), cost 0.012 (+0.20)", "move": null }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "rank": 2, "score": 0.654, "normalized": { "accuracy": 0.92, "latency_ms": 0, "cost": 0.97 }, "why": "accuracy 0.92 (+0.46), cost 0.03 (+0.19)", "move": null }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "rank": 3, "score": 0.6482, "normalized": { "accuracy": 0.9, "latency_ms": 0, "cost": 0.991 }, "why": "accuracy 0.9 (+0.45), cost 0.009 (+0.20)", "move": null }, { "id": "it_e7567efc82a32053", "label": "Rules engine", "rank": 4, "score": 0.5598, "normalized": { "accuracy": 0.72, "latency_ms": 0, "cost": 0.999 }, "why": "accuracy 0.72 (+0.36), cost 0.001 (+0.20)", "move": null } ], "excluded": [], "snapshot": "rs_2c431f2e496069ac", "notes": [ "latency_ms: some values are outside 0..1 and were clamped (normalize none)." ] } }, { "name": "rankItems better target", "fn": "rankItems", "input": { "criteria": [ { "id": "temp_c", "label": "Temperature", "weight": 1, "better": "target", "target": 21, "tolerance": 4 } ], "items": [ { "label": "Room A", "values": { "temp_c": 20 } }, { "label": "Room B", "values": { "temp_c": 23.5 } }, { "label": "Room C", "values": { "temp_c": 27 } } ] }, "output": { "weights": { "temp_c": 1 }, "items": [ { "id": "it_bae0b72009392142", "label": "Room A", "rank": 1, "score": 0.75, "normalized": { "temp_c": 0.75 }, "why": "temp_c 20 (+0.75)", "move": null }, { "id": "it_277a1f6b3bc22e18", "label": "Room B", "rank": 2, "score": 0.375, "normalized": { "temp_c": 0.375 }, "why": "temp_c 23.5 (+0.38)", "move": null }, { "id": "it_b530ebbd164cd35f", "label": "Room C", "rank": 3, "score": 0, "normalized": { "temp_c": 0 }, "why": "temp_c 27 (+0.00)", "move": null } ], "excluded": [], "snapshot": "rs_b8ec1578980408fe", "notes": [] } }, { "name": "rankItems ties competition", "fn": "rankItems", "input": { "criteria": [ { "id": "x", "label": "X", "weight": 1, "better": "higher" } ], "items": [ { "label": "A", "values": { "x": 3 } }, { "label": "B", "values": { "x": 2 } }, { "label": "C", "values": { "x": 2 } }, { "label": "D", "values": { "x": 1 } } ], "ties": "competition" }, "output": { "weights": { "x": 1 }, "items": [ { "id": "it_f949f4fb3d4cade5", "label": "A", "rank": 1, "score": 1, "normalized": { "x": 1 }, "why": "x 3 (+1.00)", "move": null }, { "id": "it_b5c66de257235791", "label": "B", "rank": 2, "score": 0.5, "normalized": { "x": 0.5 }, "why": "x 2 (+0.50)", "move": null }, { "id": "it_a2cb46ecc5978833", "label": "C", "rank": 2, "score": 0.5, "normalized": { "x": 0.5 }, "why": "x 2 (+0.50)", "move": null }, { "id": "it_e0f318ad021017a5", "label": "D", "rank": 4, "score": 0, "normalized": { "x": 0 }, "why": "x 1 (+0.00)", "move": null } ], "excluded": [], "snapshot": "rs_82ff868d2796dc7d", "notes": [] } }, { "name": "rankItems ties dense", "fn": "rankItems", "input": { "criteria": [ { "id": "x", "label": "X", "weight": 1, "better": "higher" } ], "items": [ { "label": "A", "values": { "x": 3 } }, { "label": "B", "values": { "x": 2 } }, { "label": "C", "values": { "x": 2 } }, { "label": "D", "values": { "x": 1 } } ], "ties": "dense" }, "output": { "weights": { "x": 1 }, "items": [ { "id": "it_f949f4fb3d4cade5", "label": "A", "rank": 1, "score": 1, "normalized": { "x": 1 }, "why": "x 3 (+1.00)", "move": null }, { "id": "it_b5c66de257235791", "label": "B", "rank": 2, "score": 0.5, "normalized": { "x": 0.5 }, "why": "x 2 (+0.50)", "move": null }, { "id": "it_a2cb46ecc5978833", "label": "C", "rank": 2, "score": 0.5, "normalized": { "x": 0.5 }, "why": "x 2 (+0.50)", "move": null }, { "id": "it_e0f318ad021017a5", "label": "D", "rank": 3, "score": 0, "normalized": { "x": 0 }, "why": "x 1 (+0.00)", "move": null } ], "excluded": [], "snapshot": "rs_bf4824d36a2b6214", "notes": [] } }, { "name": "rankItems ties ordinal", "fn": "rankItems", "input": { "criteria": [ { "id": "x", "label": "X", "weight": 1, "better": "higher" } ], "items": [ { "label": "A", "values": { "x": 3 } }, { "label": "B", "values": { "x": 2 } }, { "label": "C", "values": { "x": 2 } }, { "label": "D", "values": { "x": 1 } } ], "ties": "ordinal" }, "output": { "weights": { "x": 1 }, "items": [ { "id": "it_f949f4fb3d4cade5", "label": "A", "rank": 1, "score": 1, "normalized": { "x": 1 }, "why": "x 3 (+1.00)", "move": null }, { "id": "it_b5c66de257235791", "label": "B", "rank": 2, "score": 0.5, "normalized": { "x": 0.5 }, "why": "x 2 (+0.50)", "move": null }, { "id": "it_a2cb46ecc5978833", "label": "C", "rank": 3, "score": 0.5, "normalized": { "x": 0.5 }, "why": "x 2 (+0.50)", "move": null }, { "id": "it_e0f318ad021017a5", "label": "D", "rank": 4, "score": 0, "normalized": { "x": 0 }, "why": "x 1 (+0.00)", "move": null } ], "excluded": [], "snapshot": "rs_3c16c6a64a4c7563", "notes": [] } }, { "name": "rankItems ties fractional", "fn": "rankItems", "input": { "criteria": [ { "id": "x", "label": "X", "weight": 1, "better": "higher" } ], "items": [ { "label": "A", "values": { "x": 3 } }, { "label": "B", "values": { "x": 2 } }, { "label": "C", "values": { "x": 2 } }, { "label": "D", "values": { "x": 1 } } ], "ties": "fractional" }, "output": { "weights": { "x": 1 }, "items": [ { "id": "it_f949f4fb3d4cade5", "label": "A", "rank": 1, "score": 1, "normalized": { "x": 1 }, "why": "x 3 (+1.00)", "move": null }, { "id": "it_b5c66de257235791", "label": "B", "rank": 2.5, "score": 0.5, "normalized": { "x": 0.5 }, "why": "x 2 (+0.50)", "move": null }, { "id": "it_a2cb46ecc5978833", "label": "C", "rank": 2.5, "score": 0.5, "normalized": { "x": 0.5 }, "why": "x 2 (+0.50)", "move": null }, { "id": "it_e0f318ad021017a5", "label": "D", "rank": 4, "score": 0, "normalized": { "x": 0 }, "why": "x 1 (+0.00)", "move": null } ], "excluded": [], "snapshot": "rs_041e4e5df271063f", "notes": [] } }, { "name": "rankItems missing worst", "fn": "rankItems", "input": { "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio" }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 } }, { "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 } }, { "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 } }, { "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": null, "cost": 0.001 } } ], "missing": "worst" }, "output": { "weights": { "accuracy": 0.5, "latency_ms": 0.3, "cost": 0.2 }, "items": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "rank": 1, "score": 0.886638, "normalized": { "accuracy": 1, "latency_ms": 0.875, "cost": 0.62069 }, "why": "accuracy 0.94 (+0.50), latency_ms 420 (+0.26)", "move": null }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "rank": 2, "score": 0.853918, "normalized": { "accuracy": 0.818182, "latency_ms": 1, "cost": 0.724138 }, "why": "accuracy 0.9 (+0.41), latency_ms 180 (+0.30)", "move": null }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "rank": 3, "score": 0.454545, "normalized": { "accuracy": 0.909091, "latency_ms": 0, "cost": 0 }, "why": "accuracy 0.92 (+0.45), latency_ms 2100 (+0.00)", "move": null }, { "id": "it_e7567efc82a32053", "label": "Rules engine", "rank": 4, "score": 0.2, "normalized": { "accuracy": 0, "latency_ms": null, "cost": 1 }, "why": "cost 0.001 (+0.20), accuracy 0.72 (+0.00)", "move": null } ], "excluded": [], "snapshot": "rs_aad6f31886a17d36", "notes": [ "Some items are missing values; the missing rule \"worst\" applied." ] } }, { "name": "rankItems missing mean", "fn": "rankItems", "input": { "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio" }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 } }, { "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 } }, { "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 } }, { "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": null, "cost": 0.001 } } ], "missing": "mean" }, "output": { "weights": { "accuracy": 0.5, "latency_ms": 0.3, "cost": 0.2 }, "items": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "rank": 1, "score": 0.886638, "normalized": { "accuracy": 1, "latency_ms": 0.875, "cost": 0.62069 }, "why": "accuracy 0.94 (+0.50), latency_ms 420 (+0.26)", "move": null }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "rank": 2, "score": 0.853918, "normalized": { "accuracy": 0.818182, "latency_ms": 1, "cost": 0.724138 }, "why": "accuracy 0.9 (+0.41), latency_ms 180 (+0.30)", "move": null }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "rank": 3, "score": 0.454545, "normalized": { "accuracy": 0.909091, "latency_ms": 0, "cost": 0 }, "why": "accuracy 0.92 (+0.45), latency_ms 2100 (+0.00)", "move": null }, { "id": "it_e7567efc82a32053", "label": "Rules engine", "rank": 4, "score": 0.3875, "normalized": { "accuracy": 0, "latency_ms": null, "cost": 1 }, "why": "cost 0.001 (+0.20), latency_ms missing (+0.19)", "move": null } ], "excluded": [], "snapshot": "rs_005fa7e694dd17c2", "notes": [ "Some items are missing values; the missing rule \"mean\" applied." ] } }, { "name": "rankItems missing skip", "fn": "rankItems", "input": { "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio" }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 } }, { "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 } }, { "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 } }, { "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": null, "cost": 0.001 } } ], "missing": "skip" }, "output": { "weights": { "accuracy": 0.5, "latency_ms": 0.3, "cost": 0.2 }, "items": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "rank": 1, "score": 0.886638, "normalized": { "accuracy": 1, "latency_ms": 0.875, "cost": 0.62069 }, "why": "accuracy 0.94 (+0.50), latency_ms 420 (+0.26)", "move": null }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "rank": 2, "score": 0.853918, "normalized": { "accuracy": 0.818182, "latency_ms": 1, "cost": 0.724138 }, "why": "accuracy 0.9 (+0.41), latency_ms 180 (+0.30)", "move": null }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "rank": 3, "score": 0.454545, "normalized": { "accuracy": 0.909091, "latency_ms": 0, "cost": 0 }, "why": "accuracy 0.92 (+0.45), latency_ms 2100 (+0.00)", "move": null }, { "id": "it_e7567efc82a32053", "label": "Rules engine", "rank": 4, "score": 0.285714, "normalized": { "accuracy": 0, "latency_ms": null, "cost": 1 }, "why": "cost 0.001 (+0.29), accuracy 0.72 (+0.00)", "move": null } ], "excluded": [], "snapshot": "rs_b286dba903955123", "notes": [ "Some items are missing values; the missing rule \"skip\" applied." ] } }, { "name": "rankItems with a must and previous ranks", "fn": "rankItems", "input": { "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio", "must": { "op": ">=", "value": 0.8 } }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 } }, { "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 } }, { "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 } }, { "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": 40, "cost": 0.001 } } ], "previous": { "it_c3c9e33d47b2f564": 1, "it_3c2349db14c0d212": 2 } }, "output": { "weights": { "accuracy": 0.5, "latency_ms": 0.3, "cost": 0.2 }, "items": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "rank": 1, "score": 0.933929, "normalized": { "accuracy": 1, "latency_ms": 0.875, "cost": 0.857143 }, "why": "accuracy 0.94 (+0.50), latency_ms 420 (+0.26)", "move": 1 }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "rank": 2, "score": 0.5, "normalized": { "accuracy": 0, "latency_ms": 1, "cost": 1 }, "why": "latency_ms 180 (+0.30), cost 0.009 (+0.20)", "move": -1 }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "rank": 3, "score": 0.25, "normalized": { "accuracy": 0.5, "latency_ms": 0, "cost": 0 }, "why": "accuracy 0.92 (+0.25), latency_ms 2100 (+0.00)", "move": "new" } ], "excluded": [ { "id": "it_e7567efc82a32053", "label": "Rules engine", "failed": [ { "criterion": "accuracy", "value": 0.72, "must": { "op": ">=", "value": 0.8 } } ] } ], "snapshot": "rs_1af539289aea8aab", "notes": [] } }, { "name": "rankItems from decaying signals", "fn": "rankItems", "input": { "criteria": [ { "id": "success", "label": "Success", "weight": 1, "better": "higher", "unit": "ratio", "half_life": "6h", "max_age": "2d" } ], "items": [ { "label": "Tool A" }, { "label": "Tool B" } ], "signals": [ { "item": "Tool A", "criterion": "success", "value": 0.9, "at": "2026-10-24T12:00:00Z", "source": "logs", "n": 50 }, { "item": "Tool A", "criterion": "success", "value": 0.7, "at": "2026-10-24T18:00:00Z", "source": "logs", "n": 50 }, { "item": "Tool B", "criterion": "success", "value": 0.8, "at": "2026-10-24T17:00:00Z", "source": "peer", "n": 40 }, { "item": "Tool B", "criterion": "success", "value": 0.95, "at": "2026-10-20T00:00:00Z", "source": "peer", "n": 40 } ], "sources": [ { "id": "logs", "kind": "own", "trust": 1 }, { "id": "peer", "kind": "agent", "trust": 0.5 } ], "now": "2026-10-24T18:00:00Z" }, "output": { "items": [ { "label": "Tool B", "rank": 1, "score": 1, "values": { "success": 0.8 }, "stale": [] }, { "label": "Tool A", "rank": 2, "score": 0, "values": { "success": 0.766667 }, "stale": [] } ], "snapshot": "rs_b74b9c20f1e5de14", "notes": [] } }, { "name": "rankUncertainty", "fn": "rankUncertainty", "input": { "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio" }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 }, "se": { "accuracy": 0.02 } }, { "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 }, "se": { "accuracy": 0.03 } }, { "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 }, "se": { "accuracy": 0.025 } }, { "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": 40, "cost": 0.001 }, "se": { "accuracy": 0.04 } } ], "draws": 500, "seed": 7, "k": 2 }, "output": { "items": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "score": 0.868798, "low": 0.785375, "high": 0.868798, "p_best": 0.688, "p_top_k": 1, "expected_rank": 1.312, "rank_low": 1, "rank_high": 2 }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "score": 0.83353, "low": 0.696851, "high": 0.924439, "p_best": 0.312, "p_top_k": 0.998, "expected_rank": 1.69, "rank_low": 1, "rank_high": 2 }, { "id": "it_e7567efc82a32053", "label": "Rules engine", "score": 0.5, "low": 0.5, "high": 0.5, "p_best": 0, "p_top_k": 0.002, "expected_rank": 2.998, "rank_low": 3, "rank_high": 3 }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "score": 0.454545, "low": 0.344115, "high": 0.5, "p_best": 0, "p_top_k": 0, "expected_rank": 3.74, "rank_low": 3, "rank_high": 4 } ], "lead_probability": 0.688, "draws": 500, "seed": 7, "k": 2, "notes": [] } }, { "name": "decide on a ranking (default rule)", "fn": "decide", "input": { "ranking": "rankItems(criteria, items) above", "rule": {} }, "output": { "verdict": "ask", "pick": { "id": "it_3c2349db14c0d212", "label": "Layout parser", "rank": 1, "score": 0.868798 }, "runner_up": { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "rank": 2, "score": 0.83353 }, "shortlist": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "rank": 1, "score": 0.868798 }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "rank": 2, "score": 0.83353 }, { "id": "it_e7567efc82a32053", "label": "Rules engine", "rank": 3, "score": 0.5 }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "rank": 4, "score": 0.454545 } ], "lead": 0.035268, "p_best": null, "fresh": null, "fresh_until": null, "age": null, "checks": [ { "rule": "has_items", "pass": true, "detail": "4 items ranked." }, { "rule": "fresh", "pass": true, "detail": "Not checked: as_of, fresh_for or now is unknown." }, { "rule": "min_score", "pass": true, "detail": "Layout parser scores 0.869; the rule is 0.00." }, { "rule": "min_lead", "pass": false, "detail": "Layout parser leads OCR service by 0.04; the rule is 0.10." }, { "rule": "min_confidence", "pass": true, "detail": "Not checked: no p_best (run rank_uncertainty or rank_items with uncertainty)." } ], "flips": [ { "kind": "value", "criterion": "accuracy", "item": "it_c3c9e33d47b2f564", "change": 0.070536, "detail": "OCR service takes first if its accuracy rose by 0.07 on the 0..1 scale, all else equal." }, { "kind": "value", "criterion": "accuracy", "item": "it_3c2349db14c0d212", "change": -0.070536, "detail": "Layout parser loses first if its accuracy fell by 0.07 on the 0..1 scale, all else equal." }, { "kind": "value", "criterion": "latency_ms", "item": "it_3c2349db14c0d212", "change": -0.11756, "detail": "Layout parser loses first if its latency_ms fell by 0.12 on the 0..1 scale, all else equal." }, { "kind": "weight", "criterion": "accuracy", "item": "it_c3c9e33d47b2f564", "change": -0.120327, "detail": "OCR service takes first if accuracy's weight went from 0.50 to 0.38, the others scaled to match." }, { "kind": "weight", "criterion": "latency_ms", "item": "it_c3c9e33d47b2f564", "change": 0.162661, "detail": "OCR service takes first if latency_ms's weight went from 0.30 to 0.46, the others scaled to match." }, { "kind": "value", "criterion": "cost", "item": "it_c3c9e33d47b2f564", "change": 0.17634, "detail": "OCR service takes first if its cost rose by 0.18 on the 0..1 scale, all else equal." } ], "within_budget": null, "summary": "Ask before acting: Layout parser leads by 0.04, under the 0.10 rule." } }, { "name": "decide with a close lead asks", "fn": "decide", "input": { "ranking": { "items": [ { "id": "a", "label": "A", "score": 0.81 }, { "id": "b", "label": "B", "score": 0.78 }, { "id": "c", "label": "C", "score": 0.5 } ] }, "rule": { "min_lead": 0.1 } }, "output": { "verdict": "ask", "pick": { "id": "a", "label": "A", "rank": 1, "score": 0.81 }, "runner_up": { "id": "b", "label": "B", "rank": 2, "score": 0.78 }, "shortlist": [ { "id": "a", "label": "A", "rank": 1, "score": 0.81 }, { "id": "b", "label": "B", "rank": 2, "score": 0.78 }, { "id": "c", "label": "C", "rank": 3, "score": 0.5 } ], "lead": 0.03, "p_best": null, "fresh": null, "fresh_until": null, "age": null, "checks": [ { "rule": "has_items", "pass": true, "detail": "3 items ranked." }, { "rule": "fresh", "pass": true, "detail": "Not checked: as_of, fresh_for or now is unknown." }, { "rule": "min_score", "pass": true, "detail": "A scores 0.810; the rule is 0.00." }, { "rule": "min_lead", "pass": false, "detail": "A leads B by 0.03; the rule is 0.10." }, { "rule": "min_confidence", "pass": true, "detail": "Not checked: no p_best (run rank_uncertainty or rank_items with uncertainty)." } ], "flips": [], "within_budget": null, "summary": "Ask before acting: A leads by 0.03, under the 0.10 rule." } }, { "name": "decide on a stale ranking rechecks", "fn": "decide", "input": { "ranking": { "items": [ { "id": "a", "label": "A", "score": 0.81 }, { "id": "b", "label": "B", "score": 0.78 }, { "id": "c", "label": "C", "score": 0.5 } ] }, "rule": { "min_lead": 0 }, "as_of": "2026-10-24T06:00:00Z", "fresh_for": "6h", "now": "2026-10-24T18:00:00Z" }, "output": { "verdict": "recheck", "pick": { "id": "a", "label": "A", "rank": 1, "score": 0.81 }, "runner_up": { "id": "b", "label": "B", "rank": 2, "score": 0.78 }, "shortlist": [ { "id": "a", "label": "A", "rank": 1, "score": 0.81 }, { "id": "b", "label": "B", "rank": 2, "score": 0.78 }, { "id": "c", "label": "C", "rank": 3, "score": 0.5 } ], "lead": 0.03, "p_best": null, "fresh": false, "fresh_until": "2026-10-24T12:00:00Z", "age": "12h", "checks": [ { "rule": "has_items", "pass": true, "detail": "3 items ranked." }, { "rule": "fresh", "pass": false, "detail": "Stale: past 2026-10-24T12:00:00Z (as_of 2026-10-24T06:00:00Z + 6h). Rule on_stale: recheck." }, { "rule": "min_score", "pass": true, "detail": "A scores 0.810; the rule is 0.00." }, { "rule": "min_lead", "pass": true, "detail": "A leads B by 0.03; the rule is 0.00." }, { "rule": "min_confidence", "pass": true, "detail": "Not checked: no p_best (run rank_uncertainty or rank_items with uncertainty)." } ], "flips": [], "within_budget": null, "summary": "Recheck before acting: the ranking is stale (fresh until 2026-10-24T12:00:00Z); refresh the signals, rank again and decide again." } }, { "name": "decide picks within a budget", "fn": "decide", "input": { "ranking": { "items": [ { "id": "c1", "label": "Chunk 1", "score": 0.9, "values": { "tokens": 400 } }, { "id": "c2", "label": "Chunk 2", "score": 0.8, "values": { "tokens": 900 } }, { "id": "c3", "label": "Chunk 3", "score": 0.6, "values": { "tokens": 200 } }, { "id": "c4", "label": "Chunk 4", "score": 0.5, "values": { "tokens": 300 } } ] }, "rule": { "budget": 1000, "cost_of": "tokens", "min_lead": 0 } }, "output": { "verdict": "act", "pick": { "id": "c1", "label": "Chunk 1", "rank": 1, "score": 0.9 }, "runner_up": { "id": "c2", "label": "Chunk 2", "rank": 2, "score": 0.8 }, "shortlist": [ { "id": "c1", "label": "Chunk 1", "rank": 1, "score": 0.9 }, { "id": "c2", "label": "Chunk 2", "rank": 2, "score": 0.8 }, { "id": "c3", "label": "Chunk 3", "rank": 3, "score": 0.6 }, { "id": "c4", "label": "Chunk 4", "rank": 4, "score": 0.5 } ], "lead": 0.1, "p_best": null, "fresh": null, "fresh_until": null, "age": null, "checks": [ { "rule": "has_items", "pass": true, "detail": "4 items ranked." }, { "rule": "fresh", "pass": true, "detail": "Not checked: as_of, fresh_for or now is unknown." }, { "rule": "min_score", "pass": true, "detail": "Chunk 1 scores 0.900; the rule is 0.00." }, { "rule": "min_lead", "pass": true, "detail": "Chunk 1 leads Chunk 2 by 0.10; the rule is 0.00." }, { "rule": "min_confidence", "pass": true, "detail": "Not checked: no p_best (run rank_uncertainty or rank_items with uncertainty)." } ], "flips": [], "within_budget": { "budget": 1000, "cost_of": "tokens", "picks": [ { "id": "c1", "label": "Chunk 1", "score": 0.9, "cost": 400 }, { "id": "c3", "label": "Chunk 3", "score": 0.6, "cost": 200 }, { "id": "c4", "label": "Chunk 4", "score": 0.5, "cost": 300 } ], "total_cost": 900, "total_score": 2, "method": "by_score" }, "summary": "Act on Chunk 1: it leads Chunk 2 by 0.10 (rule 0.00)." } }, { "name": "decide with nothing to rank", "fn": "decide", "input": { "ranking": { "items": [] } }, "output": { "verdict": "ask", "pick": null, "runner_up": null, "shortlist": [], "lead": null, "p_best": null, "fresh": null, "fresh_until": null, "age": null, "checks": [ { "rule": "has_items", "pass": false, "detail": "No items to choose from (none given, or the musts excluded them all)." } ], "flips": [], "within_budget": null, "summary": "Ask before acting: nothing to choose from (no items, or the musts excluded every item)." } }, { "name": "recheckValue with drift", "fn": "recheckValue", "input": { "items": [ { "id": "a", "score": 0.8, "se": 0.03, "drift_per_hour": 0.01 }, { "id": "b", "score": 0.74, "se": 0.03, "drift_per_hour": 0.01 }, { "id": "c", "score": 0.5, "se": 0.02 } ], "age": "4h", "max_flip_risk": 0.1 }, "output": { "verdict": "recheck", "leader": { "id": "a", "label": "a", "score": 0.8 }, "p_flip": 0.119658, "max_flip_risk": 0.1, "challengers": [ { "id": "b", "label": "b", "gap": 0.06, "p": 0.119658 }, { "id": "c", "label": "c", "gap": 0.3, "p": 0 } ], "age": "4h", "horizon": "0s", "recheck_in": "0s", "recheck_at": null, "notes": [ "Challengers are treated as independent, so P_flip is an approximation." ], "summary": "Recheck before acting: chance the lead flips is 0.12 (rule 0.10); closest challenger b trails by 0.06. Look again now." } }, { "name": "recheckValue with costs", "fn": "recheckValue", "input": { "items": [ { "id": "a", "score": 0.8, "se": 0.01 }, { "id": "b", "score": 0.6, "se": 0.01 } ], "cost_recheck": 1, "cost_wrong": 100 }, "output": { "verdict": "act", "leader": { "id": "a", "label": "a", "score": 0.8 }, "p_flip": 0, "max_flip_risk": 0.1, "challengers": [ { "id": "b", "label": "b", "gap": 0.2, "p": 0 } ], "age": "0s", "horizon": "0s", "recheck_in": null, "recheck_at": null, "notes": [ "Challengers are treated as independent, so P_flip is an approximation.", "No drift given: the lead only changes with new signals." ], "summary": "Act on a: chance the lead flips is 0.00 (expected cost of a wrong pick 0.00 vs 1.00 to recheck); closest challenger b trails by 0.20." } }, { "name": "freshness mode exponential", "fn": "freshness", "input": { "signals": [ { "item": "x", "value": 10, "at": "2026-10-24T12:00:00Z" }, { "item": "x", "value": 4, "at": "2026-10-24T17:00:00Z" }, { "item": "y", "value": 6, "at": "2026-10-23T18:00:00Z" } ], "now": "2026-10-24T18:00:00Z", "half_life": "6h", "max_age": "1d", "mode": "exponential" }, "output": { "mode": "exponential", "now": "2026-10-24T18:00:00Z", "signals": [ { "item": "x", "value": 10, "at": "2026-10-24T12:00:00Z", "age": "6h", "age_hours": 6, "weight": 0.5, "stale": false }, { "item": "x", "value": 4, "at": "2026-10-24T17:00:00Z", "age": "1h", "age_hours": 1, "weight": 0.890899, "stale": false }, { "item": "y", "value": 6, "at": "2026-10-23T18:00:00Z", "age": "1d", "age_hours": 24, "weight": 0.0625, "stale": false } ], "items": [ { "item": "x", "rank": 1, "value": 6.156879, "signals": 2, "fresh_signals": 2, "n_eff": 1.853596, "newest": "2026-10-24T17:00:00Z", "fresh_until": "2026-10-25T17:00:00Z", "stale": false }, { "item": "y", "rank": 2, "value": 6, "signals": 1, "fresh_signals": 1, "n_eff": 1, "newest": "2026-10-23T18:00:00Z", "fresh_until": "2026-10-24T18:00:00Z", "stale": false } ], "recheck_by": "2026-10-24T18:00:00Z", "notes": [], "summary": "2 items from 3 signals (exponential); x leads with 6.1569; 0 stale; look again by 2026-10-24T18:00:00Z (now)." } }, { "name": "freshness mode linear", "fn": "freshness", "input": { "signals": [ { "item": "x", "value": 10, "at": "2026-10-24T12:00:00Z" }, { "item": "x", "value": 4, "at": "2026-10-24T17:00:00Z" }, { "item": "y", "value": 6, "at": "2026-10-23T18:00:00Z" } ], "now": "2026-10-24T18:00:00Z", "half_life": "6h", "max_age": "1d", "mode": "linear" }, "output": { "mode": "linear", "now": "2026-10-24T18:00:00Z", "signals": [ { "item": "x", "value": 10, "at": "2026-10-24T12:00:00Z", "age": "6h", "age_hours": 6, "weight": 0.75, "stale": false }, { "item": "x", "value": 4, "at": "2026-10-24T17:00:00Z", "age": "1h", "age_hours": 1, "weight": 0.958333, "stale": false }, { "item": "y", "value": 6, "at": "2026-10-23T18:00:00Z", "age": "1d", "age_hours": 24, "weight": 0, "stale": false } ], "items": [ { "item": "x", "rank": 1, "value": 6.634146, "signals": 2, "fresh_signals": 2, "n_eff": 1.970692, "newest": "2026-10-24T17:00:00Z", "fresh_until": "2026-10-25T17:00:00Z", "stale": false }, { "item": "y", "rank": 2, "value": null, "signals": 1, "fresh_signals": 0, "n_eff": 0, "newest": "2026-10-23T18:00:00Z", "fresh_until": "2026-10-24T18:00:00Z", "stale": false } ], "recheck_by": "2026-10-24T18:00:00Z", "notes": [], "summary": "2 items from 3 signals (linear); x leads with 6.6341; 0 stale; look again by 2026-10-24T18:00:00Z (now)." } }, { "name": "freshness mode window", "fn": "freshness", "input": { "signals": [ { "item": "x", "value": 10, "at": "2026-10-24T12:00:00Z" }, { "item": "x", "value": 4, "at": "2026-10-24T17:00:00Z" }, { "item": "y", "value": 6, "at": "2026-10-23T18:00:00Z" } ], "now": "2026-10-24T18:00:00Z", "half_life": "6h", "max_age": "1d", "mode": "window" }, "output": { "mode": "window", "now": "2026-10-24T18:00:00Z", "signals": [ { "item": "x", "value": 10, "at": "2026-10-24T12:00:00Z", "age": "6h", "age_hours": 6, "weight": 1, "stale": false }, { "item": "x", "value": 4, "at": "2026-10-24T17:00:00Z", "age": "1h", "age_hours": 1, "weight": 1, "stale": false }, { "item": "y", "value": 6, "at": "2026-10-23T18:00:00Z", "age": "1d", "age_hours": 24, "weight": 1, "stale": false } ], "items": [ { "item": "x", "rank": 1, "value": 7, "signals": 2, "fresh_signals": 2, "n_eff": 2, "newest": "2026-10-24T17:00:00Z", "fresh_until": "2026-10-25T17:00:00Z", "stale": false }, { "item": "y", "rank": 2, "value": 6, "signals": 1, "fresh_signals": 1, "n_eff": 1, "newest": "2026-10-23T18:00:00Z", "fresh_until": "2026-10-24T18:00:00Z", "stale": false } ], "recheck_by": "2026-10-24T18:00:00Z", "notes": [], "summary": "2 items from 3 signals (window); x leads with 7; 0 stale; look again by 2026-10-24T18:00:00Z (now)." } }, { "name": "freshness mode gravity", "fn": "freshness", "input": { "signals": [ { "item": "x", "value": 10, "at": "2026-10-24T12:00:00Z" }, { "item": "x", "value": 4, "at": "2026-10-24T17:00:00Z" }, { "item": "y", "value": 6, "at": "2026-10-23T18:00:00Z" } ], "now": "2026-10-24T18:00:00Z", "half_life": "6h", "max_age": "1d", "mode": "gravity" }, "output": { "mode": "gravity", "now": "2026-10-24T18:00:00Z", "signals": [ { "item": "x", "value": 10, "at": "2026-10-24T12:00:00Z", "age": "6h", "age_hours": 6, "weight": 0.023683, "stale": false }, { "item": "x", "value": 4, "at": "2026-10-24T17:00:00Z", "age": "1h", "age_hours": 1, "weight": 0.138415, "stale": false }, { "item": "y", "value": 6, "at": "2026-10-23T18:00:00Z", "age": "1d", "age_hours": 24, "weight": 0.002838, "stale": false } ], "items": [ { "item": "x", "rank": 1, "value": 0.790489, "signals": 2, "fresh_signals": 2, "n_eff": 1.332471, "newest": "2026-10-24T17:00:00Z", "fresh_until": "2026-10-25T17:00:00Z", "stale": false }, { "item": "y", "rank": 2, "value": 0.017029, "signals": 1, "fresh_signals": 1, "n_eff": 1, "newest": "2026-10-23T18:00:00Z", "fresh_until": "2026-10-24T18:00:00Z", "stale": false } ], "recheck_by": "2026-10-24T18:00:00Z", "notes": [], "summary": "2 items from 3 signals (gravity); x leads with 0.7905; 0 stale; look again by 2026-10-24T18:00:00Z (now)." } }, { "name": "mergeRankings rrf", "fn": "mergeRankings", "input": { "lists": [ { "source": "keyword", "items": [ "Setup guide", "API reference", "Pricing page", "Migration notes" ] }, { "source": "vector", "items": [ "API reference", "Migration notes", "Setup guide", "Error codes" ] }, { "source": "peer", "weight": 0.5, "items": [ "Migration notes", "Error codes", "API reference" ] } ], "method": "rrf" }, "output": { "fused": [ { "id": "it_0e42693b36f711cc", "label": "API reference", "rank": 1, "score": 0.040459, "ranks": [ 2, 1, 3 ], "lists": 3 }, { "id": "it_75f101f0b4b99053", "label": "Migration notes", "rank": 2, "score": 0.039951, "ranks": [ 4, 2, 1 ], "lists": 3 }, { "id": "it_1f8b6e9238921855", "label": "Setup guide", "rank": 3, "score": 0.032266, "ranks": [ 1, 3, null ], "lists": 2 }, { "id": "it_b623cf60eee61f4b", "label": "Error codes", "rank": 4, "score": 0.02369, "ranks": [ null, 4, 2 ], "lists": 2 }, { "id": "it_344b625d600ff740", "label": "Pricing page", "rank": 5, "score": 0.015873, "ranks": [ 3, null, null ], "lists": 1 } ], "agreement": { "lists": [ { "source": "keyword", "tau": 0 }, { "source": "vector", "tau": 1 }, { "source": "peer", "tau": -0.333333 } ], "mean": 0.222222, "least": { "source": "peer", "tau": -0.333333 } }, "universe": 5, "notes": [] } }, { "name": "mergeRankings borda", "fn": "mergeRankings", "input": { "lists": [ { "source": "keyword", "items": [ "Setup guide", "API reference", "Pricing page", "Migration notes" ] }, { "source": "vector", "items": [ "API reference", "Migration notes", "Setup guide", "Error codes" ] }, { "source": "peer", "weight": 0.5, "items": [ "Migration notes", "Error codes", "API reference" ] } ], "method": "borda" }, "output": { "fused": [ { "id": "it_0e42693b36f711cc", "label": "API reference", "rank": 1, "score": 8, "ranks": [ 2, 1, 3 ], "lists": 3 }, { "id": "it_1f8b6e9238921855", "label": "Setup guide", "rank": 2, "score": 6.25, "ranks": [ 1, 3, null ], "lists": 2 }, { "id": "it_75f101f0b4b99053", "label": "Migration notes", "rank": 3, "score": 6, "ranks": [ 4, 2, 1 ], "lists": 3 }, { "id": "it_b623cf60eee61f4b", "label": "Error codes", "rank": 4, "score": 2.5, "ranks": [ null, 4, 2 ], "lists": 2 }, { "id": "it_344b625d600ff740", "label": "Pricing page", "rank": 5, "score": 2.25, "ranks": [ 3, null, null ], "lists": 1 } ], "agreement": { "lists": [ { "source": "keyword", "tau": 0.333333 }, { "source": "vector", "tau": 0.666667 }, { "source": "peer", "tau": -0.333333 } ], "mean": 0.222222, "least": { "source": "peer", "tau": -0.333333 } }, "universe": 5, "notes": [] } }, { "name": "mergeRankings copeland", "fn": "mergeRankings", "input": { "lists": [ { "source": "keyword", "items": [ "Setup guide", "API reference", "Pricing page", "Migration notes" ] }, { "source": "vector", "items": [ "API reference", "Migration notes", "Setup guide", "Error codes" ] }, { "source": "peer", "weight": 0.5, "items": [ "Migration notes", "Error codes", "API reference" ] } ], "method": "copeland" }, "output": { "fused": [ { "id": "it_0e42693b36f711cc", "label": "API reference", "rank": 1, "score": 1, "ranks": [ 2, 1, 3 ], "lists": 3 }, { "id": "it_75f101f0b4b99053", "label": "Migration notes", "rank": 2, "score": 0.75, "ranks": [ 4, 2, 1 ], "lists": 3 }, { "id": "it_1f8b6e9238921855", "label": "Setup guide", "rank": 3, "score": 0.5, "ranks": [ 1, 3, null ], "lists": 2 }, { "id": "it_b623cf60eee61f4b", "label": "Error codes", "rank": 4, "score": 0.25, "ranks": [ null, 4, 2 ], "lists": 2 }, { "id": "it_344b625d600ff740", "label": "Pricing page", "rank": 5, "score": 0, "ranks": [ 3, null, null ], "lists": 1 } ], "agreement": { "lists": [ { "source": "keyword", "tau": 0 }, { "source": "vector", "tau": 1 }, { "source": "peer", "tau": -0.333333 } ], "mean": 0.222222, "least": { "source": "peer", "tau": -0.333333 } }, "universe": 5, "notes": [] } }, { "name": "mergeRankings schulze", "fn": "mergeRankings", "input": { "lists": [ { "source": "keyword", "items": [ "Setup guide", "API reference", "Pricing page", "Migration notes" ] }, { "source": "vector", "items": [ "API reference", "Migration notes", "Setup guide", "Error codes" ] }, { "source": "peer", "weight": 0.5, "items": [ "Migration notes", "Error codes", "API reference" ] } ], "method": "schulze" }, "output": { "fused": [ { "id": "it_0e42693b36f711cc", "label": "API reference", "rank": 1, "score": 1, "ranks": [ 2, 1, 3 ], "lists": 3 }, { "id": "it_75f101f0b4b99053", "label": "Migration notes", "rank": 2, "score": 0.75, "ranks": [ 4, 2, 1 ], "lists": 3 }, { "id": "it_1f8b6e9238921855", "label": "Setup guide", "rank": 3, "score": 0.5, "ranks": [ 1, 3, null ], "lists": 2 }, { "id": "it_b623cf60eee61f4b", "label": "Error codes", "rank": 4, "score": 0.25, "ranks": [ null, 4, 2 ], "lists": 2 }, { "id": "it_344b625d600ff740", "label": "Pricing page", "rank": 5, "score": 0, "ranks": [ 3, null, null ], "lists": 1 } ], "agreement": { "lists": [ { "source": "keyword", "tau": 0 }, { "source": "vector", "tau": 1 }, { "source": "peer", "tau": -0.333333 } ], "mean": 0.222222, "least": { "source": "peer", "tau": -0.333333 } }, "universe": 5, "notes": [] } }, { "name": "mergeRankings kemeny", "fn": "mergeRankings", "input": { "lists": [ { "source": "keyword", "items": [ "Setup guide", "API reference", "Pricing page", "Migration notes" ] }, { "source": "vector", "items": [ "API reference", "Migration notes", "Setup guide", "Error codes" ] }, { "source": "peer", "weight": 0.5, "items": [ "Migration notes", "Error codes", "API reference" ] } ], "method": "kemeny" }, "output": { "fused": [ { "id": "it_0e42693b36f711cc", "label": "API reference", "rank": 1, "score": 1, "ranks": [ 2, 1, 3 ], "lists": 3 }, { "id": "it_75f101f0b4b99053", "label": "Migration notes", "rank": 2, "score": 0.75, "ranks": [ 4, 2, 1 ], "lists": 3 }, { "id": "it_1f8b6e9238921855", "label": "Setup guide", "rank": 3, "score": 0.5, "ranks": [ 1, 3, null ], "lists": 2 }, { "id": "it_b623cf60eee61f4b", "label": "Error codes", "rank": 4, "score": 0.25, "ranks": [ null, 4, 2 ], "lists": 2 }, { "id": "it_344b625d600ff740", "label": "Pricing page", "rank": 5, "score": 0, "ranks": [ 3, null, null ], "lists": 1 } ], "agreement": { "lists": [ { "source": "keyword", "tau": 0 }, { "source": "vector", "tau": 1 }, { "source": "peer", "tau": -0.333333 } ], "mean": 0.222222, "least": { "source": "peer", "tau": -0.333333 } }, "universe": 5, "notes": [] } }, { "name": "mergeRankings median", "fn": "mergeRankings", "input": { "lists": [ { "source": "keyword", "items": [ "Setup guide", "API reference", "Pricing page", "Migration notes" ] }, { "source": "vector", "items": [ "API reference", "Migration notes", "Setup guide", "Error codes" ] }, { "source": "peer", "weight": 0.5, "items": [ "Migration notes", "Error codes", "API reference" ] } ], "method": "median" }, "output": { "fused": [ { "id": "it_0e42693b36f711cc", "label": "API reference", "rank": 1, "score": 0.75, "ranks": [ 2, 1, 3 ], "lists": 3 }, { "id": "it_75f101f0b4b99053", "label": "Migration notes", "rank": 1, "score": 0.75, "ranks": [ 4, 2, 1 ], "lists": 3 }, { "id": "it_1f8b6e9238921855", "label": "Setup guide", "rank": 3, "score": 0.5, "ranks": [ 1, 3, null ], "lists": 2 }, { "id": "it_b623cf60eee61f4b", "label": "Error codes", "rank": 4, "score": 0.25, "ranks": [ null, 4, 2 ], "lists": 2 }, { "id": "it_344b625d600ff740", "label": "Pricing page", "rank": 5, "score": 0.125, "ranks": [ 3, null, null ], "lists": 1 } ], "agreement": { "lists": [ { "source": "keyword", "tau": 0 }, { "source": "vector", "tau": 1 }, { "source": "peer", "tau": -0.333333 } ], "mean": 0.222222, "least": { "source": "peer", "tau": -0.333333 } }, "universe": 5, "notes": [ "median ignores list weights." ] } }, { "name": "compareRankings identical", "fn": "compareRankings", "input": { "a": [ "a", "b", "c", "d" ], "b": [ "a", "b", "c", "d" ], "p": 0.9 }, "output": { "overlap": { "common": 4, "only_a": 0, "only_b": 0, "jaccard": 1 }, "top_k": { "1": 1, "3": 1 }, "kendall_tau_b": 1, "spearman_rho": 1, "footrule": 0, "rbo": 1, "p": 0.9, "verdict": "strong agreement", "thresholds": { "strong": 0.7, "partial": 0.4, "weak": 0 }, "discordant": [], "notes": [], "formula": "Matched by it_ id, else normalized text. jaccard = common / union; top_k share = |a[:k] and b[:k]| / k; Kendall tau-b and Spearman rho on common items; footrule = sum |pos_a - pos_b| over common items re-ranked, / floor(n^2/2); RBO_ext (WMZ 2010, eq. 32) with persistence p: (1-p)/p (sum_{d<=l} (X_d/d) p^d + sum_{d=s+1..l} X_s (d-s)/(s d) p^d) + ((X_l - X_s)/l + X_s/s) p^l.", "summary": "4 common items (jaccard 1); Kendall tau-b 1, RBO 1 at p 0.9: strong agreement." } }, { "name": "compareRankings disjoint", "fn": "compareRankings", "input": { "a": [ "a", "b" ], "b": [ "c", "d" ], "p": 0.9 }, "output": { "overlap": { "common": 0, "only_a": 2, "only_b": 2, "jaccard": 0 }, "top_k": { "1": 0 }, "kendall_tau_b": null, "spearman_rho": null, "footrule": null, "rbo": 0, "p": 0.9, "verdict": "weak agreement", "thresholds": { "strong": 0.7, "partial": 0.4, "weak": 0 }, "discordant": [], "notes": [], "formula": "Matched by it_ id, else normalized text. jaccard = common / union; top_k share = |a[:k] and b[:k]| / k; Kendall tau-b and Spearman rho on common items; footrule = sum |pos_a - pos_b| over common items re-ranked, / floor(n^2/2); RBO_ext (WMZ 2010, eq. 32) with persistence p: (1-p)/p (sum_{d<=l} (X_d/d) p^d + sum_{d=s+1..l} X_s (d-s)/(s d) p^d) + ((X_l - X_s)/l + X_s/s) p^l.", "summary": "0 common items (jaccard 0); Kendall tau-b n/a, RBO 0 at p 0.9: weak agreement." } }, { "name": "compareRankings partial", "fn": "compareRankings", "input": { "a": [ "a", "b", "c", "d", "e" ], "b": [ "b", "a", "d", "f" ], "p": 0.9 }, "output": { "overlap": { "common": 3, "only_a": 2, "only_b": 1, "jaccard": 0.5 }, "top_k": { "1": 0, "3": 0.666667, "5": 0.6 }, "kendall_tau_b": 0.333333, "spearman_rho": 0.5, "footrule": 0.5, "rbo": 0.69075, "p": 0.9, "verdict": "weak agreement", "thresholds": { "strong": 0.7, "partial": 0.4, "weak": 0 }, "discordant": [ { "x": "a", "y": "b", "a": [ 1, 2 ], "b": [ 2, 1 ] } ], "notes": [], "formula": "Matched by it_ id, else normalized text. jaccard = common / union; top_k share = |a[:k] and b[:k]| / k; Kendall tau-b and Spearman rho on common items; footrule = sum |pos_a - pos_b| over common items re-ranked, / floor(n^2/2); RBO_ext (WMZ 2010, eq. 32) with persistence p: (1-p)/p (sum_{d<=l} (X_d/d) p^d + sum_{d=s+1..l} X_s (d-s)/(s d) p^d) + ((X_l - X_s)/l + X_s/s) p^l.", "summary": "3 common items (jaccard 0.5); Kendall tau-b 0.33, RBO 0.69 at p 0.9: weak agreement." } }, { "name": "rankChanges", "fn": "rankChanges", "input": { "before": [ { "id": "a", "label": "A", "rank": 1, "score": 0.9 }, { "id": "b", "label": "B", "rank": 2, "score": 0.8 }, { "id": "c", "label": "C", "rank": 3, "score": 0.7 }, { "id": "d", "label": "D", "rank": 4, "score": 0.6 } ], "after": [ { "id": "b", "label": "B", "rank": 1, "score": 0.85 }, { "id": "a", "label": "A", "rank": 2, "score": 0.84 }, { "id": "e", "label": "E", "rank": 3, "score": 0.7 }, { "id": "c", "label": "C", "rank": 4, "score": 0.65 } ], "at": "2026-10-24T18:00:00Z", "top_k": 3, "history": [ { "as_of": "2026-10-24T12:00:00Z", "ranks": { "a": 1, "b": 2, "c": 3, "d": 4 }, "scores": { "a": 0.9, "b": 0.8, "c": 0.7, "d": 0.6 } } ] }, "output": { "up": [ { "id": "b", "label": "B", "from": 2, "to": 1, "by": 1 } ], "down": [ { "id": "a", "label": "A", "from": 1, "to": 2, "by": 1 }, { "id": "c", "label": "C", "from": 3, "to": 4, "by": 1 } ], "entered": [ { "id": "e", "label": "E", "to": 3 } ], "exited": [ { "id": "d", "label": "D", "from": 4 } ], "overtakes": [ { "item": "b", "other": "a", "detail": "B passed A (2 to 1)." } ], "new_leader": { "id": "b", "label": "B", "previous": "a" }, "stability": { "kendall_tau": 0.333333, "top_k": 3, "top_k_churn": 0.333333, "kept": 2 }, "events": [ { "id": "rc_f0a29529b8e26b56", "at": "2026-10-24T18:00:00Z", "type": "new_leader", "item": "b", "from": 2, "to": 1, "detail": "B took first from A.", "other": "a" }, { "id": "rc_6bb586414e4cdcaf", "at": "2026-10-24T18:00:00Z", "type": "entered", "item": "e", "from": null, "to": 3, "detail": "E entered at 3." }, { "id": "rc_f12d530e9b9c0a54", "at": "2026-10-24T18:00:00Z", "type": "exited", "item": "d", "from": 4, "to": null, "detail": "D left (was 4)." }, { "id": "rc_fc96b03828ef5fc3", "at": "2026-10-24T18:00:00Z", "type": "moved", "item": "b", "from": 2, "to": 1, "detail": "B up 1 (2 to 1)." }, { "id": "rc_732352833dc6ec46", "at": "2026-10-24T18:00:00Z", "type": "moved", "item": "a", "from": 1, "to": 2, "detail": "A down 1 (1 to 2)." }, { "id": "rc_3ab170f914813816", "at": "2026-10-24T18:00:00Z", "type": "moved", "item": "c", "from": 3, "to": 4, "detail": "C down 1 (3 to 4)." }, { "id": "rc_102b5bd324ba716c", "at": "2026-10-24T18:00:00Z", "type": "overtook", "item": "b", "from": 2, "to": 1, "detail": "B passed A (2 to 1).", "other": "a" } ], "volatility": { "b": { "per_hour": 0.008333, "samples": 1 }, "a": { "per_hour": 0.01, "samples": 1 }, "c": { "per_hour": 0.008333, "samples": 1 } }, "summary": "B took first from A; 1 up, 2 down, 1 new, 1 out; top 3 churn 0.33." } }, { "name": "pairwiseRatings elo", "fn": "pairwiseRatings", "input": { "matches": [ { "a": "Prompt A", "b": "Prompt B", "winner": "a" }, { "a": "Prompt A", "b": "Prompt B", "winner": "a" }, { "a": "Prompt A", "b": "Prompt B", "winner": "a" }, { "a": "Prompt A", "b": "Prompt B", "winner": "b" }, { "a": "Prompt B", "b": "Prompt C", "winner": "a" }, { "a": "Prompt C", "b": "Prompt A", "winner": "tie" } ], "method": "elo" }, "output": { "method": "elo", "formula": "Elo, in the order given: expected E_a = 1 / (1 + 10^((R_b - R_a)/400)); R_a += k (S_a - E_a), S = 1 win, 0.5 tie, 0 loss.", "players": [ { "id": "it_14eae4cc4536cf96", "label": "Prompt A", "rank": 1, "rating": 1521.926294, "low": null, "high": null, "games": 5, "wins": 3, "losses": 1, "ties": 1, "win_rate": 0.7 }, { "id": "it_21abd20ac0d00485", "label": "Prompt B", "rank": 2, "rating": 1493.29342, "low": null, "high": null, "games": 5, "wins": 2, "losses": 3, "ties": 0, "win_rate": 0.4 }, { "id": "it_8841f89506afd870", "label": "Prompt C", "rank": 3, "rating": 1484.780286, "low": null, "high": null, "games": 2, "wins": 0, "losses": 1, "ties": 1, "win_rate": 0.25 } ], "p_beats": [ { "a": "Prompt A", "b": "Prompt B", "p": 0.541113 }, { "a": "Prompt A", "b": "Prompt C", "p": 0.553255 } ], "notes": [ "Elo depends on the order of the matches; send them oldest first." ], "summary": "3 players from 6 matches by elo: Prompt A leads (rating 1522, 3-1-1), beats Prompt B with p 0.54." } }, { "name": "pairwiseRatings bradley_terry", "fn": "pairwiseRatings", "input": { "matches": [ { "a": "Prompt A", "b": "Prompt B", "winner": "a" }, { "a": "Prompt A", "b": "Prompt B", "winner": "a" }, { "a": "Prompt A", "b": "Prompt B", "winner": "a" }, { "a": "Prompt A", "b": "Prompt B", "winner": "b" }, { "a": "Prompt B", "b": "Prompt C", "winner": "a" }, { "a": "Prompt C", "b": "Prompt A", "winner": "tie" } ], "method": "bradley_terry" }, "output": { "method": "bradley_terry", "formula": "Bradley-Terry by MM (Hunter 2004): pi_i = W_i / sum_j n_ij / (pi_i + pi_j), ties as half a win each, plus one virtual tie per player against a fixed reference of strength 1; geometric mean of pi normalized to 1 each pass; stop at max |delta ln pi| < 1e-9 or 1000 passes. rating = start + 400 log10(pi); se(ln pi) = 1/sqrt(sum_j n_ij p_ij (1 - p_ij)); interval +-1.645 se.", "players": [ { "id": "it_14eae4cc4536cf96", "label": "Prompt A", "rank": 1, "rating": 1600.517828, "low": 1352.226215, "high": 1848.809441, "games": 5, "wins": 3, "losses": 1, "ties": 1, "win_rate": 0.7 }, { "id": "it_21abd20ac0d00485", "label": "Prompt B", "rank": 2, "rating": 1491.686671, "low": 1249.632955, "high": 1733.740387, "games": 5, "wins": 2, "losses": 3, "ties": 0, "win_rate": 0.4 }, { "id": "it_8841f89506afd870", "label": "Prompt C", "rank": 3, "rating": 1407.795501, "low": 1054.876598, "high": 1760.714404, "games": 2, "wins": 0, "losses": 1, "ties": 1, "win_rate": 0.25 } ], "p_beats": [ { "a": "Prompt A", "b": "Prompt B", "p": 0.651691 }, { "a": "Prompt A", "b": "Prompt C", "p": 0.752017 } ], "notes": [], "summary": "3 players from 6 matches by bradley_terry: Prompt A leads (rating 1601, 3-1-1), beats Prompt B with p 0.65.", "passes": 22 } }, { "name": "pairwiseRatings win_rate", "fn": "pairwiseRatings", "input": { "matches": [ { "a": "Prompt A", "b": "Prompt B", "winner": "a" }, { "a": "Prompt A", "b": "Prompt B", "winner": "a" }, { "a": "Prompt A", "b": "Prompt B", "winner": "a" }, { "a": "Prompt A", "b": "Prompt B", "winner": "b" }, { "a": "Prompt B", "b": "Prompt C", "winner": "a" }, { "a": "Prompt C", "b": "Prompt A", "winner": "tie" } ], "method": "win_rate" }, "output": { "method": "win_rate", "formula": "Win rate = (wins + ties/2) / games; ranked by the Wilson lower bound (z 1.96); low/high = the Wilson interval.", "players": [ { "id": "it_14eae4cc4536cf96", "label": "Prompt A", "rank": 1, "rating": 0.298786, "low": 0.298786, "high": 0.927418, "games": 5, "wins": 3, "losses": 1, "ties": 1, "win_rate": 0.7 }, { "id": "it_21abd20ac0d00485", "label": "Prompt B", "rank": 2, "rating": 0.117618, "low": 0.117618, "high": 0.76928, "games": 5, "wins": 2, "losses": 3, "ties": 0, "win_rate": 0.4 }, { "id": "it_8841f89506afd870", "label": "Prompt C", "rank": 3, "rating": 0.026677, "low": 0.026677, "high": 0.802137, "games": 2, "wins": 0, "losses": 1, "ties": 1, "win_rate": 0.25 } ], "p_beats": [], "notes": [ "win_rate gives no head-to-head probability; use bradley_terry or elo for p_beats." ], "summary": "3 players from 6 matches by win_rate: Prompt A leads (Wilson lower bound 0.299, 3-1-1)." } }, { "name": "pickNext thompson", "fn": "pickNext", "input": { "options": [ { "id": "a", "successes": 30, "failures": 10 }, { "id": "b", "successes": 5, "failures": 1 }, { "id": "c", "successes": 0, "failures": 0 } ], "policy": "thompson", "seed": 3 }, "output": { "policy": "thompson", "formula": "Thompson sampling: one draw from each posterior (Beta(alpha + s, beta + f), or Normal(mean, sd^2/n)), pick the largest.", "kind": "binary", "pick": { "id": "a", "label": "a", "reason": "largest posterior draw (0.7881)" }, "mode": "explore", "options": [ { "id": "a", "label": "a", "mean": 0.738095, "low": 0.620544, "high": 0.837342, "p_best": 0.312, "trials": 40 }, { "id": "b", "label": "b", "mean": 0.75, "low": 0.491988, "high": 0.946336, "p_best": 0.4915, "trials": 6 }, { "id": "c", "label": "c", "mean": 0.5, "low": 0.047039, "high": 0.951378, "p_best": 0.1965, "trials": 0 } ], "regret_proxy": 0.011905, "seed": 3, "draws": 2000, "notes": [], "summary": "Explore: try a next (largest posterior draw (0.7881)); its chance of being best is 0.31, the best mean so far is b." } }, { "name": "pickNext ucb1", "fn": "pickNext", "input": { "options": [ { "id": "a", "successes": 30, "failures": 10 }, { "id": "b", "successes": 5, "failures": 1 }, { "id": "c", "successes": 0, "failures": 0 } ], "policy": "ucb1", "seed": 3 }, "output": { "policy": "ucb1", "formula": "UCB1: any untried option first; else the largest mean + sqrt(2 ln t / n_i) (continuous: mean + sd sqrt(2 ln t / n_i)), t = total trials.", "kind": "binary", "pick": { "id": "c", "label": "c", "reason": "untried option first" }, "mode": "explore", "options": [ { "id": "a", "label": "a", "mean": 0.738095, "low": 0.620544, "high": 0.837342, "p_best": 0.312, "trials": 40, "ucb": 1.18753 }, { "id": "b", "label": "b", "mean": 0.75, "low": 0.491988, "high": 0.946336, "p_best": 0.4915, "trials": 6, "ucb": 1.96303 }, { "id": "c", "label": "c", "mean": 0.5, "low": 0.047039, "high": 0.951378, "p_best": 0.1965, "trials": 0 } ], "regret_proxy": 0.25, "seed": 3, "draws": 2000, "notes": [], "summary": "Explore: try c next (untried option first); its chance of being best is 0.2, the best mean so far is b." } }, { "name": "pickNext epsilon_greedy", "fn": "pickNext", "input": { "options": [ { "id": "a", "successes": 30, "failures": 10 }, { "id": "b", "successes": 5, "failures": 1 }, { "id": "c", "successes": 0, "failures": 0 } ], "policy": "epsilon_greedy", "seed": 3 }, "output": { "policy": "epsilon_greedy", "formula": "Epsilon greedy: with chance epsilon a uniform pick, else the largest posterior mean.", "kind": "binary", "pick": { "id": "b", "label": "b", "reason": "best posterior mean (draw 0.72 >= epsilon 0.1)" }, "mode": "exploit", "options": [ { "id": "a", "label": "a", "mean": 0.738095, "low": 0.620544, "high": 0.837342, "p_best": 0.312, "trials": 40 }, { "id": "b", "label": "b", "mean": 0.75, "low": 0.491988, "high": 0.946336, "p_best": 0.4915, "trials": 6 }, { "id": "c", "label": "c", "mean": 0.5, "low": 0.047039, "high": 0.951378, "p_best": 0.1965, "trials": 0 } ], "regret_proxy": 0, "seed": 3, "draws": 2000, "notes": [], "summary": "Use the best: try b next (best posterior mean (draw 0.72 >= epsilon 0.1)); its chance of being best is 0.49." } }, { "name": "pickNext greedy", "fn": "pickNext", "input": { "options": [ { "id": "a", "successes": 30, "failures": 10 }, { "id": "b", "successes": 5, "failures": 1 }, { "id": "c", "successes": 0, "failures": 0 } ], "policy": "greedy", "seed": 3 }, "output": { "policy": "greedy", "formula": "Greedy: the largest posterior mean.", "kind": "binary", "pick": { "id": "b", "label": "b", "reason": "best posterior mean" }, "mode": "exploit", "options": [ { "id": "a", "label": "a", "mean": 0.738095, "low": 0.620544, "high": 0.837342, "p_best": 0.312, "trials": 40 }, { "id": "b", "label": "b", "mean": 0.75, "low": 0.491988, "high": 0.946336, "p_best": 0.4915, "trials": 6 }, { "id": "c", "label": "c", "mean": 0.5, "low": 0.047039, "high": 0.951378, "p_best": 0.1965, "trials": 0 } ], "regret_proxy": 0, "seed": 3, "draws": 2000, "notes": [], "summary": "Use the best: try b next (best posterior mean); its chance of being best is 0.49." } }, { "name": "pickNext continuous", "fn": "pickNext", "input": { "options": [ { "id": "x", "mean": 120, "sd": 30, "n": 20 }, { "id": "y", "mean": 110, "sd": 20, "n": 5 } ], "policy": "thompson", "seed": 1 }, "output": { "policy": "thompson", "formula": "Thompson sampling: one draw from each posterior (Beta(alpha + s, beta + f), or Normal(mean, sd^2/n)), pick the largest.", "kind": "continuous", "pick": { "id": "x", "label": "x", "reason": "largest posterior draw (129.4206)" }, "mode": "exploit", "options": [ { "id": "x", "label": "x", "mean": 120, "low": 109.115842, "high": 130.293437, "p_best": 0.824, "trials": 20 }, { "id": "y", "label": "y", "mean": 110, "low": 94.874849, "high": 125.087497, "p_best": 0.176, "trials": 5 } ], "regret_proxy": 0, "seed": 1, "draws": 2000, "notes": [], "summary": "Use the best: try x next (largest posterior draw (129.4206)); its chance of being best is 0.82." } }, { "name": "criteriaWeights roc", "fn": "criteriaWeights", "input": { "order": [ "accuracy", "latency_ms", "cost", "setup" ], "method": "roc" }, "output": { "weights": { "accuracy": 0.520834, "latency_ms": 0.270833, "cost": 0.145833, "setup": 0.0625 }, "method": "roc", "formula": "Rank order centroid: w_k = (1/n) sum_{i=k..n} 1/i.", "cr": null, "consistent": null, "worst_pair": null, "table": "| id | weight |\n|---|---|\n| accuracy | 0.520834 |\n| latency_ms | 0.270833 |\n| cost | 0.145833 |\n| setup | 0.0625 |", "summary": "4 weights by roc: accuracy 0.521, latency_ms 0.271, cost 0.146, setup 0.063; accuracy weighs most." } }, { "name": "criteriaWeights rank_sum", "fn": "criteriaWeights", "input": { "order": [ "accuracy", "latency_ms", "cost", "setup" ], "method": "rank_sum" }, "output": { "weights": { "accuracy": 0.4, "latency_ms": 0.3, "cost": 0.2, "setup": 0.1 }, "method": "rank_sum", "formula": "Rank sum: w_k = 2 (n + 1 - k) / (n (n + 1)).", "cr": null, "consistent": null, "worst_pair": null, "table": "| id | weight |\n|---|---|\n| accuracy | 0.4 |\n| latency_ms | 0.3 |\n| cost | 0.2 |\n| setup | 0.1 |", "summary": "4 weights by rank_sum: accuracy 0.4, latency_ms 0.3, cost 0.2, setup 0.1; accuracy weighs most." } }, { "name": "criteriaWeights reciprocal", "fn": "criteriaWeights", "input": { "order": [ "accuracy", "latency_ms", "cost", "setup" ], "method": "reciprocal" }, "output": { "weights": { "accuracy": 0.48, "latency_ms": 0.24, "cost": 0.16, "setup": 0.12 }, "method": "reciprocal", "formula": "Reciprocal: w_k = (1/k) / sum_i (1/i).", "cr": null, "consistent": null, "worst_pair": null, "table": "| id | weight |\n|---|---|\n| accuracy | 0.48 |\n| latency_ms | 0.24 |\n| cost | 0.16 |\n| setup | 0.12 |", "summary": "4 weights by reciprocal: accuracy 0.48, latency_ms 0.24, cost 0.16, setup 0.12; accuracy weighs most." } }, { "name": "criteriaWeights points", "fn": "criteriaWeights", "input": { "points": { "accuracy": 50, "latency_ms": 30, "cost": 20 } }, "output": { "weights": { "accuracy": 0.5, "latency_ms": 0.3, "cost": 0.2 }, "method": "points", "formula": "Points: w = points / sum(points).", "cr": null, "consistent": null, "worst_pair": null, "table": "| id | weight |\n|---|---|\n| accuracy | 0.5 |\n| latency_ms | 0.3 |\n| cost | 0.2 |", "summary": "3 weights by points: accuracy 0.5, latency_ms 0.3, cost 0.2; accuracy weighs most." } }, { "name": "criteriaWeights pairs (consistent)", "fn": "criteriaWeights", "input": { "pairs": [ { "a": "accuracy", "b": "latency_ms", "ratio": 2 }, { "a": "accuracy", "b": "cost", "ratio": 4 }, { "a": "latency_ms", "b": "cost", "ratio": 2 } ] }, "output": { "weights": { "accuracy": 0.571429, "latency_ms": 0.285714, "cost": 0.142857 }, "method": "ahp", "formula": "AHP: weights = principal eigenvector of the pairwise matrix (power iteration); lambda_max = mean((Aw)_i / w_i); CI = (lambda_max - n)/(n - 1); CR = CI / RI(n); consistent when CR <= 0.1.", "cr": 0, "consistent": true, "worst_pair": { "a": "accuracy", "b": "latency_ms", "ratio": 2, "implied": 2 }, "table": "| id | weight |\n|---|---|\n| accuracy | 0.571429 |\n| latency_ms | 0.285714 |\n| cost | 0.142857 |", "summary": "3 weights by ahp: accuracy 0.571, latency_ms 0.286, cost 0.143; accuracy weighs most. Consistency ratio 0 (consistent, at most 0.1)." } }, { "name": "buildRankMd", "fn": "buildRankMd", "input": { "title": "Document parsers", "subject": "tool", "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio" }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 } }, { "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 } }, { "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 } }, { "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": 40, "cost": 0.001 } } ], "as_of": "2026-10-24T18:00:00Z", "sources": [ { "id": "own_evals", "kind": "own" } ] }, "output": { "text": "---\nrank: 1\nkind: ranking\nid: rk_5a1d0b389f4ffc8f\ntitle: Document parsers\nsubject: tool\nstatus: draft\nas_of: 2026-10-24T18:00:00Z\nmethod: weighted_sum\nnormalize: minmax\nties: competition\nmissing: worst\nsnapshot: rs_b69a19c9c51cf932\n---\n\n# Document parsers\n\n## Criteria\n\n| id | label | weight | better | unit | normalize | half_life | max_age | must | how |\n|---|---|---|---|---|---|---|---|---|---|\n| accuracy | Field accuracy | 0.5 | higher | ratio | | | | | |\n| latency_ms | p95 latency | 0.3 | lower | ms | | | | | |\n| cost | Cost per call | 0.2 | lower | usd | | | | | |\n\n## Items\n\n| rank | id | label | score | low | high | move | uri | group | accuracy | latency_ms | cost |\n|---|---|---|---|---|---|---|---|---|---|---|---|\n| 1 | it_3c2349db14c0d212 | Layout parser | 0.868798 | | | | | | 0.94 | 420 | 0.012 |\n| 2 | it_c3c9e33d47b2f564 | OCR service | 0.83353 | | | | | | 0.9 | 180 | 0.009 |\n| 3 | it_e7567efc82a32053 | Rules engine | 0.5 | | | | | | 0.72 | 40 | 0.001 |\n| 4 | it_f3122d1f95fba4c3 | Vision model | 0.454545 | | | | | | 0.92 | 2100 | 0.03 |\n\n## Sources\n\n| id | kind | trust | label | uri |\n|---|---|---|---|---|\n| own_evals | own | | | |\n", "snapshot": "rs_b69a19c9c51cf932" } }, { "name": "parseRankMd of the built file", "fn": "parseRankMd", "input": "---\nrank: 1\nkind: ranking\nid: rk_5a1d0b389f4ffc8f\ntitle: Document parsers\nsubject: tool\nstatus: draft\nas_of: 2026-10-24T18:00:00Z\nmethod: weighted_sum\nnormalize: minmax\nties: competition\nmissing: worst\nsnapshot: rs_b69a19c9c51cf932\n---\n\n# Document parsers\n\n## Criteria\n\n| id | label | weight | better | unit | normalize | half_life | max_age | must | how |\n|---|---|---|---|---|---|---|---|---|---|\n| accuracy | Field accuracy | 0.5 | higher | ratio | | | | | |\n| latency_ms | p95 latency | 0.3 | lower | ms | | | | | |\n| cost | Cost per call | 0.2 | lower | usd | | | | | |\n\n## Items\n\n| rank | id | label | score | low | high | move | uri | group | accuracy | latency_ms | cost |\n|---|---|---|---|---|---|---|---|---|---|---|---|\n| 1 | it_3c2349db14c0d212 | Layout parser | 0.868798 | | | | | | 0.94 | 420 | 0.012 |\n| 2 | it_c3c9e33d47b2f564 | OCR service | 0.83353 | | | | | | 0.9 | 180 | 0.009 |\n| 3 | it_e7567efc82a32053 | Rules engine | 0.5 | | | | | | 0.72 | 40 | 0.001 |\n| 4 | it_f3122d1f95fba4c3 | Vision model | 0.454545 | | | | | | 0.92 | 2100 | 0.03 |\n\n## Sources\n\n| id | kind | trust | label | uri |\n|---|---|---|---|---|\n| own_evals | own | | | |\n", "output": { "doc": { "rank": 1, "kind": "ranking", "id": "rk_5a1d0b389f4ffc8f", "title": "Document parsers", "subject": "tool", "status": "draft", "method": "weighted_sum", "normalize": "minmax", "ties": "competition", "missing": "worst", "criteria": [ { "id": "accuracy", "label": "Field accuracy", "weight": 0.5, "better": "higher", "unit": "ratio" }, { "id": "latency_ms", "label": "p95 latency", "weight": 0.3, "better": "lower", "unit": "ms" }, { "id": "cost", "label": "Cost per call", "weight": 0.2, "better": "lower", "unit": "usd" } ], "items": [ { "id": "it_3c2349db14c0d212", "label": "Layout parser", "values": { "accuracy": 0.94, "latency_ms": 420, "cost": 0.012 }, "rank": 1, "score": 0.868798 }, { "id": "it_c3c9e33d47b2f564", "label": "OCR service", "values": { "accuracy": 0.9, "latency_ms": 180, "cost": 0.009 }, "rank": 2, "score": 0.83353 }, { "id": "it_e7567efc82a32053", "label": "Rules engine", "values": { "accuracy": 0.72, "latency_ms": 40, "cost": 0.001 }, "rank": 3, "score": 0.5 }, { "id": "it_f3122d1f95fba4c3", "label": "Vision model", "values": { "accuracy": 0.92, "latency_ms": 2100, "cost": 0.03 }, "rank": 4, "score": 0.454545 } ], "as_of": "2026-10-24T18:00:00Z", "snapshot": "rs_b69a19c9c51cf932", "sources": [ { "id": "own_evals", "kind": "own" } ] }, "errors": [], "warnings": [] } }, { "name": "scaffoldFor alert", "fn": "scaffoldFor", "input": { "subject": "alert" }, "output": { "rank": 1, "kind": "ranking", "id": "rk_6f9ad5dee552dbb9", "title": "Alerts and incidents", "subject": "alert", "question": "Which alert should be handled first?", "status": "scaffold", "fresh_for": "5m", "method": "weighted_sum", "normalize": "minmax", "ties": "competition", "missing": "worst", "decide": { "min_lead": 0, "on_stale": "recheck", "on_close": "first", "top_k": 5 }, "criteria": [ { "id": "severity", "label": "Severity", "weight": 0.3, "better": "higher", "unit": "level", "normalize": "minmax", "half_life": "30m", "max_age": "6h", "how": "Severity on your own scale, such as 1 to 5." }, { "id": "blast_radius", "label": "Blast radius", "weight": 0.2, "better": "higher", "unit": "ratio", "normalize": "none", "half_life": "15m", "max_age": "6h", "how": "Share of your systems affected, from your monitors." }, { "id": "confidence", "label": "Confidence", "weight": 0.1, "better": "higher", "unit": "ratio", "normalize": "none", "half_life": "15m", "max_age": "6h", "how": "Share of your checks that confirm the alert is real." }, { "id": "minutes_since", "label": "Minutes since raised", "weight": 0.1, "better": "lower", "unit": "minutes", "normalize": "log", "half_life": "15m", "max_age": "6h", "how": "Minutes since the alert was raised." }, { "id": "customer_impact", "label": "Customer impact", "weight": 0.2, "better": "higher", "unit": "ratio", "normalize": "none", "half_life": "15m", "max_age": "6h", "how": "Share of customer requests failing, from your own monitors." }, { "id": "duplicates", "label": "Duplicates", "weight": 0.1, "better": "lower", "unit": "count", "normalize": "minmax", "half_life": "15m", "max_age": "6h", "how": "Other open alerts that are the same issue." } ], "items": [], "notes": "Scaffold for \"Alerts and incidents\": the criteria, weights, freshness and rule for acting, with no items yet.\n\nFill it from your own data: one Items row per option with a value in each criterion column, or time-stamped signals with a source, ranked with rank_items or written with build_rank_md. Measure each criterion as its how column says. Never invent a value.\n\nBefore you act, run decide (and recheck_value when the ranking is old). A ranking supports a decision; it never authorizes an action by itself.\n\nItems for this board are added as they are built, at https://rank.md/rankings/alert.md.\n\nCheck back: send If-None-Match with the ETag of this response (304 when nothing changed), read https://rank.md/changelog.json?since=2026-10-25, or call whats_new." } }, { "name": "parseRankMd value cells and escaped pipes", "fn": "parseRankMd", "input": "---\nrank: 1\ntitle: Cells\n---\n\n# Cells\n\n## Criteria\n\n| id | label | weight | better |\n|---|---|---|---|\n| acc | Accuracy | 1 | higher |\n\n## Items\n\n| label | acc |\n|---|---|\n| A \\| B | 0.94 ±0.02 n=40 |\n| C | - |\n", "output": { "doc": { "rank": 1, "kind": "ranking", "id": "rk_7445587f20dfff45", "title": "Cells", "subject": "other", "status": "draft", "method": "weighted_sum", "normalize": "minmax", "ties": "competition", "missing": "worst", "criteria": [ { "id": "acc", "label": "Accuracy", "weight": 1, "better": "higher" } ], "items": [ { "id": "it_926f411973d8d65a", "label": "A | B", "values": { "acc": 0.94 }, "se": { "acc": 0.02 }, "n": { "acc": 40 } }, { "id": "it_a2cb46ecc5978833", "label": "C", "values": { "acc": null } } ] }, "errors": [], "warnings": [] } }, { "name": "checkRankMd of the built file, with now", "fn": "checkRankMd", "input": { "text": "---\nrank: 1\nkind: ranking\nid: rk_5a1d0b389f4ffc8f\ntitle: Document parsers\nsubject: tool\nstatus: draft\nas_of: 2026-10-24T18:00:00Z\nmethod: weighted_sum\nnormalize: minmax\nties: competition\nmissing: worst\nsnapshot: rs_b69a19c9c51cf932\n---\n\n# Document parsers\n\n## Criteria\n\n| id | label | weight | better | unit | normalize | half_life | max_age | must | how |\n|---|---|---|---|---|---|---|---|---|---|\n| accuracy | Field accuracy | 0.5 | higher | ratio | | | | | |\n| latency_ms | p95 latency | 0.3 | lower | ms | | | | | |\n| cost | Cost per call | 0.2 | lower | usd | | | | | |\n\n## Items\n\n| rank | id | label | score | low | high | move | uri | group | accuracy | latency_ms | cost |\n|---|---|---|---|---|---|---|---|---|---|---|---|\n| 1 | it_3c2349db14c0d212 | Layout parser | 0.868798 | | | | | | 0.94 | 420 | 0.012 |\n| 2 | it_c3c9e33d47b2f564 | OCR service | 0.83353 | | | | | | 0.9 | 180 | 0.009 |\n| 3 | it_e7567efc82a32053 | Rules engine | 0.5 | | | | | | 0.72 | 40 | 0.001 |\n| 4 | it_f3122d1f95fba4c3 | Vision model | 0.454545 | | | | | | 0.92 | 2100 | 0.03 |\n\n## Sources\n\n| id | kind | trust | label | uri |\n|---|---|---|---|---|\n| own_evals | own | | | |\n", "now": "2026-10-25T18:00:00Z" }, "output": { "ok": true, "errors": [], "warnings": [], "snapshot": { "listed": "rs_b69a19c9c51cf932", "recomputed": "rs_b69a19c9c51cf932", "matches": true }, "recomputed": { "matches": true, "mismatches": [], "notes": [] }, "freshness": { "as_of": "2026-10-24T18:00:00Z", "fresh_for": null, "fresh_until": null, "now": "2026-10-25T18:00:00Z", "fresh": null, "age": "1d" } } }, { "name": "checkRankMd error codes", "fn": "checkRankMd", "input": { "text": "---\nrank: 2\ntitle: Broken\nsubject: weather\n---\n\n## Criteria\n\n| id | label | weight | better |\n|---|---|---|---|\n| Bad Id | Race | -1 | up |\n\n## Items\n\n| label | x |\n|---|---|\n| A | abc |\n| B |\n" }, "output": { "ok": false, "codes": [ "E02", "E05", "E10", "E18", "E15", "E08", "R01", "E09", "E09" ], "warnings": [ "W09", "W11" ] } }, { "name": "parseRankMd with a node (rank.md 2.0)", "fn": "parseRankMd", "input": "---\nrank: 1\ntitle: Code review agents\nsubject: agent\nnode: ai-agents/agents-to-hire/code-review-agents\n---\n\n# Code review agents\n\n## Criteria\n\n| id | label | weight | better |\n|---|---|---|---|\n| catch_rate | Real bugs caught | 1 | higher |\n\n## Items\n\n| label | catch_rate |\n|---|---|\n", "output": { "doc": { "rank": 1, "kind": "ranking", "id": "rk_571c4cc4389f12e9", "title": "Code review agents", "subject": "agent", "status": "draft", "method": "weighted_sum", "normalize": "minmax", "ties": "competition", "missing": "worst", "criteria": [ { "id": "catch_rate", "label": "Real bugs caught", "weight": 1, "better": "higher" } ], "items": [], "node": "ai-agents/agents-to-hire/code-review-agents" }, "errors": [], "warnings": [] } } ] }