Patch 5: Replace silent truncation with explicit token limit in read-text-file
Silent truncation of legal documents is unacceptable - Merlyn was reading partial files without knowing it. New behavior: - Files under AGENT_MAX_FILE_TOKENS (default 500,000): return full content - Files over limit: return explicit message with token count and options - Paralegal controls any summarization decision - Merlyn never silently truncates legal documents
This commit is contained in:
parent
5bcef7d604
commit
acbf66c5bf
@ -104,19 +104,23 @@ module.exports.FilesystemReadTextFile = {
|
||||
this.super.introspect(`Successfully read ${filePath}`);
|
||||
}
|
||||
|
||||
const { content: finalContent, wasTruncated } =
|
||||
filesystem.truncateContentForContext(
|
||||
content,
|
||||
this.super,
|
||||
"[Content truncated - file exceeds context limit. Use head/tail parameters to read specific portions.]"
|
||||
);
|
||||
const { TokenManager } = require("../../../../helpers/tiktoken");
|
||||
const tokenManager = new TokenManager(this.super.model);
|
||||
const tokenCount = tokenManager.countFromString(content);
|
||||
const maxFileTokens = Number(process.env.AGENT_MAX_FILE_TOKENS) || 500_000;
|
||||
|
||||
if (wasTruncated) {
|
||||
this.super.introspect(
|
||||
`${this.caller}: File content was truncated to fit context limit`
|
||||
);
|
||||
if (tokenCount > maxFileTokens) {
|
||||
return [
|
||||
`File "${filePath}" contains ${tokenCount.toLocaleString()} tokens, which exceeds the limit of ${maxFileTokens.toLocaleString()} tokens.`,
|
||||
`To proceed, choose one of the following options:`,
|
||||
`1. Read specific portions using the head or tail parameters`,
|
||||
`2. Search for specific content using search-files`,
|
||||
`3. Request a summary of the file`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
const finalContent = content;
|
||||
|
||||
const filename = path.basename(validPath);
|
||||
this.super.addCitation?.({
|
||||
id: `fs-${Buffer.from(validPath).toString("base64url").slice(0, 32)}`,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user