The Atlas BigLaw / Big Michael — documentation bound to its code
7 documents

Submit a task to final answer

Follow a legal task from the moment it is submitted over REST or MCP, through the workflow's ordered phases, into per-phase DyTopo rounds, and out as a single Opus-synthesised answer.

src/protocols/index.ts337 lines · applyCitationGate L42–78
Outline 10 symbols
1// SPDX-License-Identifier: AGPL-3.0-only
2// Copyright (C) 2026 Discover Legal
3// This program is free software: you can redistribute it and/or modify it
4// under the terms of the GNU Affero General Public License as published by
5// the Free Software Foundation, either version 3 of the License, or
6// (at your option) any later version. See <https://www.gnu.org/licenses/>.
7
8/**
9 * Debate and verification protocols — inspired by Laverne (github.com/AnttiHero/lavern).
10 *
11 * Three layers:
12 * 1. CitationGate — no finding passes without a verifiable citation
13 * 2. DebateProtocol — adversarial challenge mechanism (challenger must also cite)
14 * 3. VerificationPipeline — 10-pass quality check on resolved findings
15 *
16 * HumanGate — findings below confidence threshold or with unresolved challenges
17 * are held pending human approval before entering final output.
18 */
19
20import { Config } from "../config.js";
21import { logger } from "../logger.js";
22import { selectModel } from "../routing/model.js";
23import { getProvider, resolveModelId, isOllamaModel, isLocalModel } from "../providers/index.js";
24import { auditLogger, ACTOR_SYSTEM } from "../audit/index.js";
25import { costStore, calcCostUsd, calcWattHours } from "../cost/index.js";
26import { sanitizePromptContent } from "../adapters/lavern.js";
27import type {
28 Finding,
29 Citation,
30 Challenge,
31 GateRequest,
32 VerificationCheck,
33 VerificationResult,
34} from "../types.js";
35
36// ─── 1. Citation gate ─────────────────────────────────────────────────────────
37
38/**
39 * Filter out findings that have no citations (required if citationRequired is true).
40 * Mechanically verify that the cited quote string appears in the source text.
41 */
42export function applyCitationGate(
43 findings: Finding[],
44 sourceTexts: Map<string, string>, // documentId → full text
45): { passed: Finding[]; rejected: Finding[] } {
46 if (!Config.debate.citationRequired) {
47 return { passed: findings, rejected: [] };
48 }
49
50 const passed: Finding[] = [];
51 const rejected: Finding[] = [];
52
53 for (const finding of findings) {
54 if (!finding.citations.length) {
55 logger.warn("Finding rejected — no citations", { findingId: finding.id });
56 rejected.push(finding);
57 continue;
58 }
59
60 // Mechanical string match
61 for (const citation of finding.citations) {
62 const sourceText = sourceTexts.get(citation.source);
63 if (sourceText) {
64 citation.mechanicallyVerified = sourceText.includes(citation.quote);
65 if (!citation.mechanicallyVerified) {
66 logger.warn("Citation string match failed", {
67 findingId: finding.id,
68 source: citation.source,
69 });
70 }
71 }
72 }
73
74 passed.push(finding);
75 }
76
77 return { passed, rejected };
78}
79
80// ─── 2. Debate protocol ───────────────────────────────────────────────────────
81
82const CHALLENGER_SYSTEM = `You are the Adversarial Challenger in a legal AI debate protocol.
83Your job: challenge the finding below if it is incorrect, overstated, or unsupported.
84Your challenge MUST include a verbatim citation from a specific source.
85If you believe the finding is correct and well-supported, output: NO_CHALLENGE
86Otherwise output:
87CHALLENGE:
88Content: <your challenge>
89Citation: SOURCE=<source> | QUOTE=<verbatim text>
90Strength: <1-5>
91END_CHALLENGE`;
92
93const RESOLVER_SYSTEM = `You are the Debate Resolver in a legal AI debate protocol.
94You receive a finding and a challenge to that finding.
95Weigh both. Cite specific reasons for your resolution.
96Output:
97RESOLUTION: <UPHELD | MODIFIED | OVERTURNED>
98REASONING: <one paragraph explaining your resolution, citing both sides>
99MODIFIED_CONTENT: <if MODIFIED, the corrected finding content; otherwise leave blank>`;
100
101export async function runDebate(finding: Finding, challengerAgentId: string, taskId?: string): Promise<Finding> {
102 if (!Config.debate.adversarialEnabled) return finding;
103
104 const debateModel = selectModel({ taskType: "debate" });
105 auditLogger.write({ event: "debate.start", actorId: ACTOR_SYSTEM, data: { findingId: finding.id, model: debateModel } });
106
107 // Cap finding content before LLM insertion to prevent oversized findings from
108 // consuming entire context windows or being used as prompt injection payloads.
109 const findingSnippet = finding.content.slice(0, 20_000);
110 // Generate challenge — Opus (debate routing)
111 const challengeText = await callModel(
112 CHALLENGER_SYSTEM,
113 `FINDING:\n${findingSnippet}\n\nCITATIONS:\n${finding.citations.slice(0, 50).map((c) => `SOURCE=${c.source.slice(0, 200)} | QUOTE=${c.quote.slice(0, 500)}`).join("\n")}`,
114 600,
115 debateModel,
116 { taskId },
117 );
118
119 if (challengeText.includes("NO_CHALLENGE")) {
120 logger.debug("Finding unchallenged", { findingId: finding.id });
121 auditLogger.write({ event: "debate.resolved", actorId: ACTOR_SYSTEM, data: { findingId: finding.id, verdict: "NO_CHALLENGE" } });
122 return finding;
123 }
124
125 const challenge = parseChallenge(challengeText, challengerAgentId);
126 finding.challenged = true;
127 finding.challenge = challenge;
128
129 // Resolve debate — Opus
130 const resolutionText = await callModel(
131 RESOLVER_SYSTEM,
132 `FINDING:\n${findingSnippet}\n\nCHALLENGE:\n${challenge.content.slice(0, 10_000)}\nChallenge citations: ${challenge.citations.map((c) => c.quote).join("; ")}`,
133 800,
134 debateModel,
135 { taskId },
136 );
137
138 const resolution = parseResolution(resolutionText);
139 challenge.resolution = resolution.reasoning;
140 challenge.resolvedAt = new Date();
141
142 if (resolution.parseError) {
143 logger.warn("Debate resolution parse error — leaving finding unresolved", { findingId: finding.id });
144 finding.resolved = false;
145 } else if (resolution.verdict === "MODIFIED" && resolution.modifiedContent) {
146 finding.content = sanitizePromptContent(resolution.modifiedContent);
147 finding.resolved = true;
148 } else if (resolution.verdict === "OVERTURNED") {
149 finding.confidence = Math.max(0, finding.confidence - 0.3);
150 finding.resolved = true;
151 } else {
152 finding.resolved = true;
153 }
154 logger.info("Debate resolved", { findingId: finding.id, verdict: resolution.verdict });
155 auditLogger.write({ event: "debate.resolved", actorId: ACTOR_SYSTEM, data: { findingId: finding.id, verdict: resolution.verdict } });
156
157 return finding;
158}
159
160// ─── 3. Verification pipeline ─────────────────────────────────────────────────
161
162const VERIFICATION_CHECKS = [
163 "Context: Is the finding grounded in the stated context and not taken out of scope?",
164 "Accuracy: Are all legal propositions correctly stated per the cited authority?",
165 "Completeness: Does the finding address all aspects of the question it purports to answer?",
166 "Clarity: Is the finding expressed clearly and unambiguously?",
167 "Structure: Is the finding logically structured?",
168 "Citations: Are all citations present, specific, and relevant?",
169 "Risk: Does the finding appropriately flag relevant risks or uncertainties?",
170 "Jurisdiction: Is the jurisdictional scope of the finding explicitly stated?",
171 "Timeliness: Are the sources current? Are any materials superseded?",
172 "Proportionality: Is the conclusion proportionate to the evidence cited?",
173];
174
175export async function runVerificationPipeline(finding: Finding, taskId?: string): Promise<VerificationResult> {
176 const passes = Math.max(1, Config.debate.verificationPasses);
177 const checksToRun = VERIFICATION_CHECKS.slice(0, passes);
178
179 const verifyModel = selectModel({ taskType: "extraction" }); // Haiku — fast, many parallel calls
180 auditLogger.write({ event: "verification.start", actorId: ACTOR_SYSTEM, data: { findingId: finding.id, checks: checksToRun.length, model: verifyModel } });
181
182 // Same 20k cap as the debate path — prevents each of the N parallel
183 // verification calls from receiving an unbounded finding payload.
184 const verifySnippet = finding.content.slice(0, 20_000);
185 const checks: VerificationCheck[] = await Promise.all(
186 checksToRun.map(async (checkDesc) => {
187 const response = await callModel(
188 `You are a legal verification specialist. Assess the following finding against this criterion: ${checkDesc}\nRespond with: PASS or FAIL followed by a one-line note.`,
189 `FINDING:\n${verifySnippet}\n\nCITATIONS:\n${finding.citations.slice(0, 50).map((c) => `${c.source.slice(0, 200)}: "${c.quote.slice(0, 500)}"`).join("\n")}`,
190 150,
191 verifyModel,
192 { taskId },
193 );
194 const passed = response.toUpperCase().includes("PASS");
195 const notes = response.replace(/^(PASS|FAIL)\s*/i, "").trim();
196 return { name: checkDesc.split(":")[0], passed, notes };
197 }),
198 );
199
200 const passed = checks.every((c) => c.passed);
201 const result: VerificationResult = {
202 findingId: finding.id,
203 checks,
204 passed,
205 completedAt: new Date(),
206 };
207
208 finding.verificationResult = result;
209 // NOTE: do not write `finding.resolved` here. `resolved` is owned by the debate
210 // protocol (runDebate) and signals whether a *challenge* was settled. Verification
211 // is an independent gate; identifyGateRequests already inspects
212 // `verificationResult.passed` separately. Overwriting `resolved` with the
213 // verification verdict previously let a passing verification mask an unresolved
214 // debate (e.g. a parse error) and skip the human gate.
215
216 const failedChecks = checks.filter((c) => !c.passed).map((c) => c.name);
217 logger.info("Verification complete", { findingId: finding.id, passed, failedChecks });
218 auditLogger.write({ event: "verification.complete", actorId: ACTOR_SYSTEM, data: { findingId: finding.id, passed, failedChecks } });
219
220 return result;
221}
222
223// ─── 4. Human gate ────────────────────────────────────────────────────────────
224
225/**
226 * Identify findings that require human review before entering final output.
227 * Criteria: low confidence OR failed verification OR unresolved challenge.
228 */
229export function identifyGateRequests(taskId: string, findings: Finding[]): GateRequest[] {
230 const threshold = Config.debate.gateConfidenceThreshold;
231 const gates: GateRequest[] = [];
232
233 for (const finding of findings) {
234 const needsGate =
235 finding.confidence < threshold ||
236 (finding.challenged && !finding.resolved) ||
237 finding.verificationResult?.passed === false;
238
239 if (needsGate) {
240 gates.push({
241 id: crypto.randomUUID(),
242 taskId,
243 findingId: finding.id,
244 finding,
245 status: "pending",
246 createdAt: new Date(),
247 });
248 }
249 }
250
251 if (gates.length) {
252 logger.info("Human gate requests created", { count: gates.length, taskId });
253 }
254
255 return gates;
256}
257
258// ─── Utility ──────────────────────────────────────────────────────────────────
259
260async function callModel(
261 system: string,
262 user: string,
263 maxTokens: number,
264 model?: string,
265 opts?: { thinking?: { budgetTokens: number }; taskId?: string },
266): Promise<string> {
267 const m = model ?? Config.anthropic.model;
268 const provider = getProvider(m);
269 const response = await provider.chat({
270 model: resolveModelId(m),
271 maxTokens,
272 system,
273 messages: [{ role: "user", content: user }],
274 cacheSystem: true,
275 ...(opts?.thinking && { thinking: opts.thinking }),
276 });
277 const isLocal = isOllamaModel(m) || isLocalModel(m);
278 const bare = resolveModelId(m);
279 const cw = response.usage.cacheWriteTokens ?? 0;
280 const cr = response.usage.cacheReadTokens ?? 0;
281 costStore.record({
282 model: bare,
283 provider: isLocal ? (isOllamaModel(m) ? "ollama" : "local") : "anthropic",
284 inputTokens: response.usage.inputTokens,
285 outputTokens: response.usage.outputTokens,
286 ...(cw ? { cacheWriteTokens: cw } : {}),
287 ...(cr ? { cacheReadTokens: cr } : {}),
288 costUsd: isLocal ? null : calcCostUsd(bare, response.usage.inputTokens, response.usage.outputTokens, cw, cr),
289 estimatedWh: isLocal ? calcWattHours(Config.local.inferenceWatts, response.durationMs) : null,
290 estimatedWatts: isLocal ? Config.local.inferenceWatts : null,
291 durationMs: response.durationMs,
292 context: maxTokens < 600 && !opts?.thinking ? "protocol_verify" : "protocol_debate",
293 taskId: opts?.taskId,
294 });
295 const block = response.content.find((b) => b.type === "text");
296 if (!block || block.type !== "text") throw new Error("Unexpected content type from model");
297 return block.text;
298}
299
300function parseChallenge(text: string, challengerId: string): Challenge {
301 const contentMatch = text.match(/Content:\s*([\s\S]+?)(?=Citation:|Strength:|END_CHALLENGE)/i);
302 const citationMatch = text.match(/Citation:\s*SOURCE=(.+?)\s*\|\s*QUOTE=(.+?)(?=\n|END_CHALLENGE)/i);
303 const strengthMatch = text.match(/Strength:\s*(\d)/i);
304
305 const citations: Citation[] = citationMatch
306 ? [{ source: citationMatch[1].trim(), quote: citationMatch[2].trim(), mechanicallyVerified: false }]
307 : [];
308
309 return {
310 challengerId,
311 challengerName: "Adversarial Challenger",
312 content: contentMatch?.[1]?.trim() ?? text,
313 citations,
314 };
315}
316
317function parseResolution(text: string): {
318 verdict: "UPHELD" | "MODIFIED" | "OVERTURNED";
319 reasoning: string;
320 modifiedContent?: string;
321 parseError?: boolean;
322} {
323 const verdictMatch = text.match(/RESOLUTION:\s*(UPHELD|MODIFIED|OVERTURNED)/i);
324 const reasoningMatch = text.match(/REASONING:\s*([\s\S]+?)(?=MODIFIED_CONTENT:|$)/i);
325 const modifiedMatch = text.match(/MODIFIED_CONTENT:\s*([\s\S]+)/i);
326
327 if (!verdictMatch) {
328 logger.warn("parseResolution: no RESOLUTION keyword found in resolver output");
329 return { verdict: "UPHELD" as const, reasoning: "Parse error - no verdict found", parseError: true };
330 }
331
332 return {
333 verdict: verdictMatch[1].toUpperCase() as "UPHELD" | "MODIFIED" | "OVERTURNED",
334 reasoning: reasoningMatch?.[1]?.trim() ?? "",
335 modifiedContent: modifiedMatch?.[1]?.trim() || undefined,
336 };
337}