fix(agent): stop tool thrash after writes and log rounds
CI / checks (push) Has been cancelled

Cap get_api_docs, break duplicate tool rounds, force a reply after
successful writes, and log each tool call so limit hits are diagnosable.
This commit is contained in:
ginnoir
2026-07-08 22:51:31 -05:00
parent 9bf8e508c7
commit 72d123b9a0
7 changed files with 389 additions and 13 deletions
@@ -4,6 +4,7 @@ export type AssistantChatMessage = {
role: "user" | "assistant"; role: "user" | "assistant";
content: string; content: string;
imageUrl?: string; imageUrl?: string;
toolCalls?: Array<{ name: string; status: number }>;
}; };
const STORAGE_VERSION = "v2"; const STORAGE_VERSION = "v2";
@@ -19,6 +20,14 @@ function isValidMessage(value: unknown): value is AssistantChatMessage {
if (row.role !== "user" && row.role !== "assistant") return false; if (row.role !== "user" && row.role !== "assistant") return false;
if (typeof row.content !== "string" || row.content.trim().length === 0) return false; if (typeof row.content !== "string" || row.content.trim().length === 0) return false;
if (row.imageUrl !== undefined && typeof row.imageUrl !== "string") return false; if (row.imageUrl !== undefined && typeof row.imageUrl !== "string") return false;
if (row.toolCalls !== undefined) {
if (!Array.isArray(row.toolCalls)) return false;
for (const call of row.toolCalls) {
if (!call || typeof call !== "object") return false;
const entry = call as Record<string, unknown>;
if (typeof entry.name !== "string" || typeof entry.status !== "number") return false;
}
}
return true; return true;
} }
@@ -229,7 +229,11 @@ export function AssistantPanel({ configured, userId, assistantName, assistantMod
setMessages((current) => [ setMessages((current) => [
...current, ...current,
{ role: "assistant", content: result.message.content }, {
role: "assistant",
content: result.message.content,
...(result.toolCalls.length > 0 ? { toolCalls: result.toolCalls } : {}),
},
]); ]);
scrollToBottom(); scrollToBottom();
} catch (err) { } catch (err) {
@@ -347,6 +351,13 @@ export function AssistantPanel({ configured, userId, assistantName, assistantMod
/> />
) : null} ) : null}
{message.content} {message.content}
{message.role === "assistant" &&
message.toolCalls &&
message.toolCalls.length > 0 ? (
<p className="mt-1 text-[10px] text-muted-foreground">
{message.toolCalls.map((call) => `${call.name}${call.status}`).join(" · ")}
</p>
) : null}
</div> </div>
))} ))}
+85
View File
@@ -0,0 +1,85 @@
const MUTATION_TOOLS = new Set([
"add_list_item",
"update_list_item",
"delete_list_item",
"create_list",
"update_list",
"delete_list",
"create_event",
"update_event",
"delete_event",
"create_calendar",
"create_note",
"update_note",
"delete_note",
"create_journal_entry",
"update_journal_entry",
"delete_journal_entry",
"create_garden_plant",
"update_garden_plant",
"delete_garden_plant",
"create_garden_container",
"update_garden_container",
"delete_garden_container",
"log_garden_care",
"create_garden_care_schedule",
"update_garden_care_schedule",
"delete_garden_care_schedule",
"schedule_garden_care_on_calendar",
"create_bang",
"update_bang",
"delete_bang",
"create_share_link",
"revoke_share_link",
]);
export function isMutationTool(name: string): boolean {
return MUTATION_TOOLS.has(name);
}
export function isSuccessfulWrite(name: string, status: number, argsJson = ""): boolean {
if (!isSuccessfulStatus(status)) return false;
if (isMutationTool(name)) return true;
if (name !== "call_api") return false;
try {
const args = JSON.parse(argsJson || "{}") as { method?: string };
const method = typeof args.method === "string" ? args.method.toUpperCase() : "";
return method === "POST" || method === "PATCH" || method === "DELETE";
} catch {
return false;
}
}
export function isSuccessfulStatus(status: number): boolean {
return status >= 200 && status < 300;
}
export function fingerprintToolCalls(calls: Array<{ name: string; arguments: string }>): string {
return calls.map((call) => `${call.name}:${normalizeArgs(call.arguments)}`).join("|");
}
export function truncateToolResult(result: string, maxChars = 6000): string {
if (result.length <= maxChars) return result;
return `${result.slice(0, maxChars)}\n…[truncated ${result.length - maxChars} chars]`;
}
export function summarizeToolTrace(toolCalls: Array<{ name: string; status: number }>): string {
if (toolCalls.length === 0) return "No tools were called.";
return toolCalls.map((call) => `${call.name}${call.status}`).join(", ");
}
function normalizeArgs(argsJson: string): string {
try {
const parsed: unknown = JSON.parse(argsJson || "{}");
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
return JSON.stringify(parsed);
}
const sorted: Record<string, unknown> = {};
for (const key of Object.keys(parsed as Record<string, unknown>).sort()) {
sorted[key] = (parsed as Record<string, unknown>)[key];
}
return JSON.stringify(sorted);
} catch {
return argsJson.trim();
}
}
+115 -3
View File
@@ -1,9 +1,16 @@
import logger from "@/lib/logger";
import { createLlmClient, type ChatMessage, type LlmClient } from "@/lib/llm"; import { createLlmClient, type ChatMessage, type LlmClient } from "@/lib/llm";
import { buildVisionContentParts, textFromMessageContent } from "@/lib/llm/content"; import { buildVisionContentParts, textFromMessageContent } from "@/lib/llm/content";
import { AGENT_SYSTEM_PROMPT, AGENT_TOOLS, appendAgentRuntimeContext } from "../tools"; import { AGENT_SYSTEM_PROMPT, AGENT_TOOLS, appendAgentRuntimeContext } from "../tools";
import type { ClientChatMessage } from "../messages"; import type { ClientChatMessage } from "../messages";
import { describeToolActivity } from "../tool-labels"; import { describeToolActivity } from "../tool-labels";
import { createApiToolExecutor, type ToolExecutor } from "../tool-executor"; import { createApiToolExecutor, type ToolExecutor } from "../tool-executor";
import {
fingerprintToolCalls,
isSuccessfulWrite,
summarizeToolTrace,
truncateToolResult,
} from "./loop-guards";
import { resolveAssistantImageDataUrls } from "./resolve-images"; import { resolveAssistantImageDataUrls } from "./resolve-images";
import { thinkingLabel, type AgentProgressEvent } from "./progress"; import { thinkingLabel, type AgentProgressEvent } from "./progress";
@@ -41,6 +48,21 @@ async function toLlmUserMessage(message: ClientChatMessage): Promise<ChatMessage
}; };
} }
function forceReplyAfterWrite(writeNames: string[]): ChatMessage {
return {
role: "user",
content: `The write already succeeded (${writeNames.join(", ")}). Stop calling tools and reply to the user in one short sentence confirming what you did.`,
};
}
function forceReplyAfterRepeat(): ChatMessage {
return {
role: "user",
content:
"You repeated the same tool call. Stop calling tools and reply with what you already know, or ask one clarifying question.",
};
}
export async function runAgentChat(options: { export async function runAgentChat(options: {
messages: ClientChatMessage[]; messages: ClientChatMessage[];
request: Request; request: Request;
@@ -62,13 +84,16 @@ export async function runAgentChat(options: {
const transcript: ChatMessage[] = [{ role: "system", content: systemPrompt }, ...userMessages]; const transcript: ChatMessage[] = [{ role: "system", content: systemPrompt }, ...userMessages];
const toolCalls: AgentToolCallSummary[] = []; const toolCalls: AgentToolCallSummary[] = [];
const seenFingerprints = new Set<string>();
const successfulWriteNames: string[] = [];
let forceReply = false;
for (let round = 0; round < MAX_TOOL_ROUNDS; round += 1) { for (let round = 0; round < MAX_TOOL_ROUNDS; round += 1) {
onProgress?.({ type: "thinking", label: thinkingLabel(round), round }); onProgress?.({ type: "thinking", label: thinkingLabel(round), round });
const completion = await llm.chatCompletion({ const completion = await llm.chatCompletion({
messages: transcript, messages: transcript,
tools: AGENT_TOOLS, tools: forceReply ? undefined : AGENT_TOOLS,
}); });
const assistantMessage = completion.message; const assistantMessage = completion.message;
@@ -76,6 +101,15 @@ export async function runAgentChat(options: {
if (!assistantMessage.tool_calls?.length) { if (!assistantMessage.tool_calls?.length) {
onProgress?.({ type: "responding", label: "Writing a reply…" }); onProgress?.({ type: "responding", label: "Writing a reply…" });
logger.info(
{
msg: "agent.chat.done",
rounds: round + 1,
toolCalls: toolCalls.map((call) => `${call.name}:${call.status}`),
forcedReply: forceReply,
},
"agent chat completed",
);
return { return {
message: { message: {
role: "assistant", role: "assistant",
@@ -87,6 +121,36 @@ export async function runAgentChat(options: {
}; };
} }
if (forceReply) {
logger.warn(
{
msg: "agent.chat.forced_tools_ignored",
round,
names: assistantMessage.tool_calls.map((c) => c.function.name),
},
"model kept calling tools after force-reply",
);
break;
}
const fingerprint = fingerprintToolCalls(
assistantMessage.tool_calls.map((call) => ({
name: call.function.name,
arguments: call.function.arguments,
})),
);
if (seenFingerprints.has(fingerprint)) {
logger.warn(
{ msg: "agent.chat.duplicate_tools", round, fingerprint, toolCalls: fingerprint },
"duplicate tool round detected",
);
transcript.push(forceReplyAfterRepeat());
forceReply = true;
continue;
}
seenFingerprints.add(fingerprint);
for (const toolCall of assistantMessage.tool_calls) { for (const toolCall of assistantMessage.tool_calls) {
const label = describeToolActivity(toolCall.function.name, toolCall.function.arguments); const label = describeToolActivity(toolCall.function.name, toolCall.function.arguments);
onProgress?.({ type: "tool", name: toolCall.function.name, label }); onProgress?.({ type: "tool", name: toolCall.function.name, label });
@@ -107,21 +171,69 @@ export async function runAgentChat(options: {
} }
toolCalls.push({ name: toolCall.function.name, status }); toolCalls.push({ name: toolCall.function.name, status });
logger.info(
{
msg: "agent.chat.tool",
round,
name: toolCall.function.name,
status,
args: toolCall.function.arguments.slice(0, 300),
},
"agent tool call",
);
if (isSuccessfulWrite(toolCall.function.name, status, toolCall.function.arguments)) {
successfulWriteNames.push(toolCall.function.name);
}
transcript.push({ transcript.push({
role: "tool", role: "tool",
tool_call_id: toolCall.id, tool_call_id: toolCall.id,
name: toolCall.function.name, name: toolCall.function.name,
content: result, content: truncateToolResult(result),
}); });
} }
if (successfulWriteNames.length > 0) {
transcript.push(forceReplyAfterWrite(successfulWriteNames));
forceReply = true;
}
} }
onProgress?.({ type: "responding", label: "Wrapping up…" }); onProgress?.({ type: "responding", label: "Wrapping up…" });
const trace = summarizeToolTrace(toolCalls);
if (successfulWriteNames.length > 0) {
logger.warn(
{
msg: "agent.chat.forced_summary",
rounds: MAX_TOOL_ROUNDS,
toolCalls: toolCalls.map((call) => `${call.name}:${call.status}`),
},
"agent summarizing after write without clean stop",
);
return {
message: {
role: "assistant",
content: `Done. Completed: ${[...new Set(successfulWriteNames)].join(", ")}.`,
},
toolCalls,
};
}
logger.warn(
{
msg: "agent.chat.tool_limit",
rounds: MAX_TOOL_ROUNDS,
toolCalls: toolCalls.map((call) => `${call.name}:${call.status}`),
forcedReply: forceReply,
},
"agent hit tool-call limit",
);
return { return {
message: { message: {
role: "assistant", role: "assistant",
content: "I hit the tool-call limit for this request. Please try a simpler question.", content: `I hit the tool-call limit for this request. Tools used: ${trace}.`,
}, },
toolCalls, toolCalls,
}; };
+9 -4
View File
@@ -451,13 +451,18 @@ async function getApiDocs(args: Record<string, unknown>): Promise<ApiCallResult>
const specPath = path.join(process.cwd(), "docs", "api", "openapi.yaml"); const specPath = path.join(process.cwd(), "docs", "api", "openapi.yaml");
const spec = await readFile(specPath, "utf8"); const spec = await readFile(specPath, "utf8");
const search = typeof args.search === "string" ? args.search.trim().toLowerCase() : ""; const search = typeof args.search === "string" ? args.search.trim().toLowerCase() : "";
const pathLines = spec
.split("\n")
.map((line) => line.trim())
.filter((line) => line.startsWith("/api/v1/"));
if (!search) { if (!search) {
return { return {
status: 200, status: 200,
body: { body: {
spec, paths: pathLines.slice(0, 80),
hint: "Pass search to filter paths, or use call_api with a /api/v1/* path.", pathCount: pathLines.length,
hint: "Pass search (e.g. calendar, events, lists) to get matching lines. Do not request the full OpenAPI dump.",
}, },
}; };
} }
@@ -469,8 +474,8 @@ async function getApiDocs(args: Record<string, unknown>): Promise<ApiCallResult>
body: { body: {
search, search,
matchCount: matches.length, matchCount: matches.length,
matches: matches.slice(0, 100), matches: matches.slice(0, 40),
hint: "Use call_api with method and path from the matches above.", hint: "Use a dedicated tool when one exists; otherwise call_api with method and path from the matches above.",
}, },
}; };
} }
+7 -5
View File
@@ -718,15 +718,16 @@ export const AGENT_TOOLS: AgentToolDefinition[] = [
function: { function: {
name: "get_api_docs", name: "get_api_docs",
description: description:
"Read famapp REST API documentation (OpenAPI). Use when unsure which endpoint to call or no dedicated tool exists. Pass search to filter relevant paths.", "Search famapp REST API docs for /api/v1 paths. Always pass search. Returns matching lines only — not the full OpenAPI file.",
parameters: { parameters: {
type: "object", type: "object",
properties: { properties: {
search: { search: {
type: "string", type: "string",
description: "Optional keyword to filter paths (e.g. garden, share, journal)", description: "Keyword to filter paths (e.g. garden, share, journal, events)",
}, },
}, },
required: ["search"],
}, },
}, },
}, },
@@ -767,10 +768,11 @@ export const AGENT_SYSTEM_PROMPT = `You are the famapp household assistant. Help
Use the provided tools to read and update data. Prefer calling tools instead of guessing. Be concise and friendly. Use the provided tools to read and update data. Prefer calling tools instead of guessing. Be concise and friendly.
Prefer a dedicated tool when one exists (create_event, add_list_item, create_note, etc.). When no dedicated tool fits, or you need an endpoint that is not wrapped yet: Prefer a dedicated tool when one exists (create_event, add_list_item, create_note, etc.). After a successful write (2xx), stop calling tools and confirm in one short sentence — do not re-list or re-create.
1. Call get_api_docs with a relevant search term to find the right /api/v1/* endpoint. When no dedicated tool fits, or you need an endpoint that is not wrapped yet:
1. Call get_api_docs with a relevant search term (always pass search; never request the full spec).
2. Call call_api with the documented method, path, query, and body. 2. Call call_api with the documented method, path, query, and body.
If a tool call fails, read the error body and fix the arguments before trying a different approach. If a tool call fails, read the error body and fix the arguments before trying a different approach. Do not repeat the exact same tool call.
When the user sends a photo, read dates, times, locations, and action items from it, then use tools to act. When the user sends a photo, read dates, times, locations, and action items from it, then use tools to act.
+152
View File
@@ -0,0 +1,152 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import {
fingerprintToolCalls,
isSuccessfulWrite,
summarizeToolTrace,
truncateToolResult,
} from "../../src/modules/agent/server/loop-guards";
import { runAgentChat } from "../../src/modules/agent/server/run";
import type {
ChatCompletionRequest,
ChatCompletionResult,
LlmClient,
} from "../../src/lib/llm/types";
describe("loop-guards", () => {
it("fingerprints tool calls stably regardless of key order", () => {
const a = fingerprintToolCalls([
{ name: "create_event", arguments: '{"title":"Dentist","calendarName":"Family"}' },
]);
const b = fingerprintToolCalls([
{ name: "create_event", arguments: '{"calendarName":"Family","title":"Dentist"}' },
]);
assert.equal(a, b);
});
it("treats create_event 201 as a successful write", () => {
assert.equal(isSuccessfulWrite("create_event", 201), true);
assert.equal(isSuccessfulWrite("list_calendars", 200), false);
assert.equal(
isSuccessfulWrite("call_api", 201, '{"method":"POST","path":"/api/v1/events"}'),
true,
);
assert.equal(
isSuccessfulWrite("call_api", 200, '{"method":"GET","path":"/api/v1/events"}'),
false,
);
});
it("truncates oversized tool results", () => {
const result = truncateToolResult("x".repeat(7000), 100);
assert.ok(result.length < 200);
assert.match(result, /truncated/);
});
it("summarizes tool traces", () => {
assert.equal(
summarizeToolTrace([
{ name: "list_calendars", status: 200 },
{ name: "create_event", status: 201 },
]),
"list_calendars→200, create_event→201",
);
});
});
describe("runAgentChat loop guards", () => {
it("stops after a successful write instead of looping", async () => {
let calls = 0;
const llm: LlmClient = {
async chatCompletion(request: ChatCompletionRequest): Promise<ChatCompletionResult> {
calls += 1;
if (calls === 1) {
return {
message: {
role: "assistant",
content: null,
tool_calls: [
{
id: "1",
type: "function",
function: {
name: "create_event",
arguments: JSON.stringify({
title: "Dentist",
startAt: "2026-07-10T15:00:00.000Z",
endAt: "2026-07-10T16:00:00.000Z",
}),
},
},
],
},
finishReason: "tool_calls",
};
}
assert.equal(request.tools, undefined);
return {
message: { role: "assistant", content: "Added Dentist to your calendar." },
finishReason: "stop",
};
},
};
const result = await runAgentChat({
messages: [{ role: "user", content: "add dentist tomorrow at 10" }],
request: new Request("http://localhost:3000/api/agent/chat"),
llm,
executeTool: async () =>
JSON.stringify({ status: 201, body: { id: "evt-1", title: "Dentist" } }),
});
assert.equal(calls, 2);
assert.equal(result.toolCalls.length, 1);
assert.equal(result.toolCalls[0]?.name, "create_event");
assert.match(result.message.content, /Dentist/);
});
it("breaks duplicate identical tool rounds", async () => {
let calls = 0;
const llm: LlmClient = {
async chatCompletion(): Promise<ChatCompletionResult> {
calls += 1;
if (calls <= 2) {
return {
message: {
role: "assistant",
content: null,
tool_calls: [
{
id: String(calls),
type: "function",
function: {
name: "list_calendars",
arguments: "{}",
},
},
],
},
finishReason: "tool_calls",
};
}
return {
message: { role: "assistant", content: "You have one Family calendar." },
finishReason: "stop",
};
},
};
const result = await runAgentChat({
messages: [{ role: "user", content: "what calendars do I have?" }],
request: new Request("http://localhost:3000/api/agent/chat"),
llm,
executeTool: async () =>
JSON.stringify({ status: 200, body: [{ id: "cal-1", name: "Family" }] }),
});
assert.equal(calls, 3);
assert.equal(result.toolCalls.length, 1);
assert.match(result.message.content, /Family/);
});
});