fix(agent): stop tool thrash after writes and log rounds
CI / checks (push) Has been cancelled

Cap get_api_docs, break duplicate tool rounds, force a reply after
successful writes, and log each tool call so limit hits are diagnosable.
This commit is contained in:
ginnoir
2026-07-08 22:51:31 -05:00
parent 9bf8e508c7
commit 72d123b9a0
7 changed files with 389 additions and 13 deletions
@@ -4,6 +4,7 @@ export type AssistantChatMessage = {
role: "user" | "assistant";
content: string;
imageUrl?: string;
toolCalls?: Array<{ name: string; status: number }>;
};
const STORAGE_VERSION = "v2";
@@ -19,6 +20,14 @@ function isValidMessage(value: unknown): value is AssistantChatMessage {
if (row.role !== "user" && row.role !== "assistant") return false;
if (typeof row.content !== "string" || row.content.trim().length === 0) return false;
if (row.imageUrl !== undefined && typeof row.imageUrl !== "string") return false;
if (row.toolCalls !== undefined) {
if (!Array.isArray(row.toolCalls)) return false;
for (const call of row.toolCalls) {
if (!call || typeof call !== "object") return false;
const entry = call as Record<string, unknown>;
if (typeof entry.name !== "string" || typeof entry.status !== "number") return false;
}
}
return true;
}
@@ -229,7 +229,11 @@ export function AssistantPanel({ configured, userId, assistantName, assistantMod
setMessages((current) => [
...current,
{ role: "assistant", content: result.message.content },
{
role: "assistant",
content: result.message.content,
...(result.toolCalls.length > 0 ? { toolCalls: result.toolCalls } : {}),
},
]);
scrollToBottom();
} catch (err) {
@@ -347,6 +351,13 @@ export function AssistantPanel({ configured, userId, assistantName, assistantMod
/>
) : null}
{message.content}
{message.role === "assistant" &&
message.toolCalls &&
message.toolCalls.length > 0 ? (
<p className="mt-1 text-[10px] text-muted-foreground">
{message.toolCalls.map((call) => `${call.name}${call.status}`).join(" · ")}
</p>
) : null}
</div>
))}
+85
View File
@@ -0,0 +1,85 @@
const MUTATION_TOOLS = new Set([
"add_list_item",
"update_list_item",
"delete_list_item",
"create_list",
"update_list",
"delete_list",
"create_event",
"update_event",
"delete_event",
"create_calendar",
"create_note",
"update_note",
"delete_note",
"create_journal_entry",
"update_journal_entry",
"delete_journal_entry",
"create_garden_plant",
"update_garden_plant",
"delete_garden_plant",
"create_garden_container",
"update_garden_container",
"delete_garden_container",
"log_garden_care",
"create_garden_care_schedule",
"update_garden_care_schedule",
"delete_garden_care_schedule",
"schedule_garden_care_on_calendar",
"create_bang",
"update_bang",
"delete_bang",
"create_share_link",
"revoke_share_link",
]);
export function isMutationTool(name: string): boolean {
return MUTATION_TOOLS.has(name);
}
export function isSuccessfulWrite(name: string, status: number, argsJson = ""): boolean {
if (!isSuccessfulStatus(status)) return false;
if (isMutationTool(name)) return true;
if (name !== "call_api") return false;
try {
const args = JSON.parse(argsJson || "{}") as { method?: string };
const method = typeof args.method === "string" ? args.method.toUpperCase() : "";
return method === "POST" || method === "PATCH" || method === "DELETE";
} catch {
return false;
}
}
export function isSuccessfulStatus(status: number): boolean {
return status >= 200 && status < 300;
}
export function fingerprintToolCalls(calls: Array<{ name: string; arguments: string }>): string {
return calls.map((call) => `${call.name}:${normalizeArgs(call.arguments)}`).join("|");
}
export function truncateToolResult(result: string, maxChars = 6000): string {
if (result.length <= maxChars) return result;
return `${result.slice(0, maxChars)}\n…[truncated ${result.length - maxChars} chars]`;
}
export function summarizeToolTrace(toolCalls: Array<{ name: string; status: number }>): string {
if (toolCalls.length === 0) return "No tools were called.";
return toolCalls.map((call) => `${call.name}${call.status}`).join(", ");
}
function normalizeArgs(argsJson: string): string {
try {
const parsed: unknown = JSON.parse(argsJson || "{}");
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
return JSON.stringify(parsed);
}
const sorted: Record<string, unknown> = {};
for (const key of Object.keys(parsed as Record<string, unknown>).sort()) {
sorted[key] = (parsed as Record<string, unknown>)[key];
}
return JSON.stringify(sorted);
} catch {
return argsJson.trim();
}
}
+115 -3
View File
@@ -1,9 +1,16 @@
import logger from "@/lib/logger";
import { createLlmClient, type ChatMessage, type LlmClient } from "@/lib/llm";
import { buildVisionContentParts, textFromMessageContent } from "@/lib/llm/content";
import { AGENT_SYSTEM_PROMPT, AGENT_TOOLS, appendAgentRuntimeContext } from "../tools";
import type { ClientChatMessage } from "../messages";
import { describeToolActivity } from "../tool-labels";
import { createApiToolExecutor, type ToolExecutor } from "../tool-executor";
import {
fingerprintToolCalls,
isSuccessfulWrite,
summarizeToolTrace,
truncateToolResult,
} from "./loop-guards";
import { resolveAssistantImageDataUrls } from "./resolve-images";
import { thinkingLabel, type AgentProgressEvent } from "./progress";
@@ -41,6 +48,21 @@ async function toLlmUserMessage(message: ClientChatMessage): Promise<ChatMessage
};
}
function forceReplyAfterWrite(writeNames: string[]): ChatMessage {
return {
role: "user",
content: `The write already succeeded (${writeNames.join(", ")}). Stop calling tools and reply to the user in one short sentence confirming what you did.`,
};
}
function forceReplyAfterRepeat(): ChatMessage {
return {
role: "user",
content:
"You repeated the same tool call. Stop calling tools and reply with what you already know, or ask one clarifying question.",
};
}
export async function runAgentChat(options: {
messages: ClientChatMessage[];
request: Request;
@@ -62,13 +84,16 @@ export async function runAgentChat(options: {
const transcript: ChatMessage[] = [{ role: "system", content: systemPrompt }, ...userMessages];
const toolCalls: AgentToolCallSummary[] = [];
const seenFingerprints = new Set<string>();
const successfulWriteNames: string[] = [];
let forceReply = false;
for (let round = 0; round < MAX_TOOL_ROUNDS; round += 1) {
onProgress?.({ type: "thinking", label: thinkingLabel(round), round });
const completion = await llm.chatCompletion({
messages: transcript,
tools: AGENT_TOOLS,
tools: forceReply ? undefined : AGENT_TOOLS,
});
const assistantMessage = completion.message;
@@ -76,6 +101,15 @@ export async function runAgentChat(options: {
if (!assistantMessage.tool_calls?.length) {
onProgress?.({ type: "responding", label: "Writing a reply…" });
logger.info(
{
msg: "agent.chat.done",
rounds: round + 1,
toolCalls: toolCalls.map((call) => `${call.name}:${call.status}`),
forcedReply: forceReply,
},
"agent chat completed",
);
return {
message: {
role: "assistant",
@@ -87,6 +121,36 @@ export async function runAgentChat(options: {
};
}
if (forceReply) {
logger.warn(
{
msg: "agent.chat.forced_tools_ignored",
round,
names: assistantMessage.tool_calls.map((c) => c.function.name),
},
"model kept calling tools after force-reply",
);
break;
}
const fingerprint = fingerprintToolCalls(
assistantMessage.tool_calls.map((call) => ({
name: call.function.name,
arguments: call.function.arguments,
})),
);
if (seenFingerprints.has(fingerprint)) {
logger.warn(
{ msg: "agent.chat.duplicate_tools", round, fingerprint, toolCalls: fingerprint },
"duplicate tool round detected",
);
transcript.push(forceReplyAfterRepeat());
forceReply = true;
continue;
}
seenFingerprints.add(fingerprint);
for (const toolCall of assistantMessage.tool_calls) {
const label = describeToolActivity(toolCall.function.name, toolCall.function.arguments);
onProgress?.({ type: "tool", name: toolCall.function.name, label });
@@ -107,21 +171,69 @@ export async function runAgentChat(options: {
}
toolCalls.push({ name: toolCall.function.name, status });
logger.info(
{
msg: "agent.chat.tool",
round,
name: toolCall.function.name,
status,
args: toolCall.function.arguments.slice(0, 300),
},
"agent tool call",
);
if (isSuccessfulWrite(toolCall.function.name, status, toolCall.function.arguments)) {
successfulWriteNames.push(toolCall.function.name);
}
transcript.push({
role: "tool",
tool_call_id: toolCall.id,
name: toolCall.function.name,
content: result,
content: truncateToolResult(result),
});
}
if (successfulWriteNames.length > 0) {
transcript.push(forceReplyAfterWrite(successfulWriteNames));
forceReply = true;
}
}
onProgress?.({ type: "responding", label: "Wrapping up…" });
const trace = summarizeToolTrace(toolCalls);
if (successfulWriteNames.length > 0) {
logger.warn(
{
msg: "agent.chat.forced_summary",
rounds: MAX_TOOL_ROUNDS,
toolCalls: toolCalls.map((call) => `${call.name}:${call.status}`),
},
"agent summarizing after write without clean stop",
);
return {
message: {
role: "assistant",
content: `Done. Completed: ${[...new Set(successfulWriteNames)].join(", ")}.`,
},
toolCalls,
};
}
logger.warn(
{
msg: "agent.chat.tool_limit",
rounds: MAX_TOOL_ROUNDS,
toolCalls: toolCalls.map((call) => `${call.name}:${call.status}`),
forcedReply: forceReply,
},
"agent hit tool-call limit",
);
return {
message: {
role: "assistant",
content: "I hit the tool-call limit for this request. Please try a simpler question.",
content: `I hit the tool-call limit for this request. Tools used: ${trace}.`,
},
toolCalls,
};
+9 -4
View File
@@ -451,13 +451,18 @@ async function getApiDocs(args: Record<string, unknown>): Promise<ApiCallResult>
const specPath = path.join(process.cwd(), "docs", "api", "openapi.yaml");
const spec = await readFile(specPath, "utf8");
const search = typeof args.search === "string" ? args.search.trim().toLowerCase() : "";
const pathLines = spec
.split("\n")
.map((line) => line.trim())
.filter((line) => line.startsWith("/api/v1/"));
if (!search) {
return {
status: 200,
body: {
spec,
hint: "Pass search to filter paths, or use call_api with a /api/v1/* path.",
paths: pathLines.slice(0, 80),
pathCount: pathLines.length,
hint: "Pass search (e.g. calendar, events, lists) to get matching lines. Do not request the full OpenAPI dump.",
},
};
}
@@ -469,8 +474,8 @@ async function getApiDocs(args: Record<string, unknown>): Promise<ApiCallResult>
body: {
search,
matchCount: matches.length,
matches: matches.slice(0, 100),
hint: "Use call_api with method and path from the matches above.",
matches: matches.slice(0, 40),
hint: "Use a dedicated tool when one exists; otherwise call_api with method and path from the matches above.",
},
};
}
+7 -5
View File
@@ -718,15 +718,16 @@ export const AGENT_TOOLS: AgentToolDefinition[] = [
function: {
name: "get_api_docs",
description:
"Read famapp REST API documentation (OpenAPI). Use when unsure which endpoint to call or no dedicated tool exists. Pass search to filter relevant paths.",
"Search famapp REST API docs for /api/v1 paths. Always pass search. Returns matching lines only — not the full OpenAPI file.",
parameters: {
type: "object",
properties: {
search: {
type: "string",
description: "Optional keyword to filter paths (e.g. garden, share, journal)",
description: "Keyword to filter paths (e.g. garden, share, journal, events)",
},
},
required: ["search"],
},
},
},
@@ -767,10 +768,11 @@ export const AGENT_SYSTEM_PROMPT = `You are the famapp household assistant. Help
Use the provided tools to read and update data. Prefer calling tools instead of guessing. Be concise and friendly.
Prefer a dedicated tool when one exists (create_event, add_list_item, create_note, etc.). When no dedicated tool fits, or you need an endpoint that is not wrapped yet:
1. Call get_api_docs with a relevant search term to find the right /api/v1/* endpoint.
Prefer a dedicated tool when one exists (create_event, add_list_item, create_note, etc.). After a successful write (2xx), stop calling tools and confirm in one short sentence — do not re-list or re-create.
When no dedicated tool fits, or you need an endpoint that is not wrapped yet:
1. Call get_api_docs with a relevant search term (always pass search; never request the full spec).
2. Call call_api with the documented method, path, query, and body.
If a tool call fails, read the error body and fix the arguments before trying a different approach.
If a tool call fails, read the error body and fix the arguments before trying a different approach. Do not repeat the exact same tool call.
When the user sends a photo, read dates, times, locations, and action items from it, then use tools to act.
+152
View File
@@ -0,0 +1,152 @@
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import {
fingerprintToolCalls,
isSuccessfulWrite,
summarizeToolTrace,
truncateToolResult,
} from "../../src/modules/agent/server/loop-guards";
import { runAgentChat } from "../../src/modules/agent/server/run";
import type {
ChatCompletionRequest,
ChatCompletionResult,
LlmClient,
} from "../../src/lib/llm/types";
describe("loop-guards", () => {
it("fingerprints tool calls stably regardless of key order", () => {
const a = fingerprintToolCalls([
{ name: "create_event", arguments: '{"title":"Dentist","calendarName":"Family"}' },
]);
const b = fingerprintToolCalls([
{ name: "create_event", arguments: '{"calendarName":"Family","title":"Dentist"}' },
]);
assert.equal(a, b);
});
it("treats create_event 201 as a successful write", () => {
assert.equal(isSuccessfulWrite("create_event", 201), true);
assert.equal(isSuccessfulWrite("list_calendars", 200), false);
assert.equal(
isSuccessfulWrite("call_api", 201, '{"method":"POST","path":"/api/v1/events"}'),
true,
);
assert.equal(
isSuccessfulWrite("call_api", 200, '{"method":"GET","path":"/api/v1/events"}'),
false,
);
});
it("truncates oversized tool results", () => {
const result = truncateToolResult("x".repeat(7000), 100);
assert.ok(result.length < 200);
assert.match(result, /truncated/);
});
it("summarizes tool traces", () => {
assert.equal(
summarizeToolTrace([
{ name: "list_calendars", status: 200 },
{ name: "create_event", status: 201 },
]),
"list_calendars→200, create_event→201",
);
});
});
describe("runAgentChat loop guards", () => {
it("stops after a successful write instead of looping", async () => {
let calls = 0;
const llm: LlmClient = {
async chatCompletion(request: ChatCompletionRequest): Promise<ChatCompletionResult> {
calls += 1;
if (calls === 1) {
return {
message: {
role: "assistant",
content: null,
tool_calls: [
{
id: "1",
type: "function",
function: {
name: "create_event",
arguments: JSON.stringify({
title: "Dentist",
startAt: "2026-07-10T15:00:00.000Z",
endAt: "2026-07-10T16:00:00.000Z",
}),
},
},
],
},
finishReason: "tool_calls",
};
}
assert.equal(request.tools, undefined);
return {
message: { role: "assistant", content: "Added Dentist to your calendar." },
finishReason: "stop",
};
},
};
const result = await runAgentChat({
messages: [{ role: "user", content: "add dentist tomorrow at 10" }],
request: new Request("http://localhost:3000/api/agent/chat"),
llm,
executeTool: async () =>
JSON.stringify({ status: 201, body: { id: "evt-1", title: "Dentist" } }),
});
assert.equal(calls, 2);
assert.equal(result.toolCalls.length, 1);
assert.equal(result.toolCalls[0]?.name, "create_event");
assert.match(result.message.content, /Dentist/);
});
it("breaks duplicate identical tool rounds", async () => {
let calls = 0;
const llm: LlmClient = {
async chatCompletion(): Promise<ChatCompletionResult> {
calls += 1;
if (calls <= 2) {
return {
message: {
role: "assistant",
content: null,
tool_calls: [
{
id: String(calls),
type: "function",
function: {
name: "list_calendars",
arguments: "{}",
},
},
],
},
finishReason: "tool_calls",
};
}
return {
message: { role: "assistant", content: "You have one Family calendar." },
finishReason: "stop",
};
},
};
const result = await runAgentChat({
messages: [{ role: "user", content: "what calendars do I have?" }],
request: new Request("http://localhost:3000/api/agent/chat"),
llm,
executeTool: async () =>
JSON.stringify({ status: 200, body: [{ id: "cal-1", name: "Family" }] }),
});
assert.equal(calls, 3);
assert.equal(result.toolCalls.length, 1);
assert.match(result.message.content, /Family/);
});
});