Patch 5: Replace silent truncation with explicit token limit in read-text-file

Silent truncation of legal documents is unacceptable - Merlyn was reading
partial files without knowing it.

New behavior:
- Files under AGENT_MAX_FILE_TOKENS (default 500,000): return full content
- Files over limit: return explicit message with token count and options
- Paralegal controls any summarization decision
- Merlyn never silently truncates legal documents
This commit is contained in:
PQ32 Developer 2026-05-10 15:28:14 -07:00
parent 5bcef7d604
commit acbf66c5bf

View File

@ -104,19 +104,23 @@ module.exports.FilesystemReadTextFile = {
this.super.introspect(`Successfully read ${filePath}`);
}
const { content: finalContent, wasTruncated } =
filesystem.truncateContentForContext(
content,
this.super,
"[Content truncated - file exceeds context limit. Use head/tail parameters to read specific portions.]"
);
const { TokenManager } = require("../../../../helpers/tiktoken");
const tokenManager = new TokenManager(this.super.model);
const tokenCount = tokenManager.countFromString(content);
const maxFileTokens = Number(process.env.AGENT_MAX_FILE_TOKENS) || 500_000;
if (wasTruncated) {
this.super.introspect(
`${this.caller}: File content was truncated to fit context limit`
);
if (tokenCount > maxFileTokens) {
return [
`File "${filePath}" contains ${tokenCount.toLocaleString()} tokens, which exceeds the limit of ${maxFileTokens.toLocaleString()} tokens.`,
`To proceed, choose one of the following options:`,
`1. Read specific portions using the head or tail parameters`,
`2. Search for specific content using search-files`,
`3. Request a summary of the file`,
].join("\n");
}
const finalContent = content;
const filename = path.basename(validPath);
this.super.addCitation?.({
id: `fs-${Buffer.from(validPath).toString("base64url").slice(0, 32)}`,