Cap get_api_docs, break duplicate tool rounds, force a reply after successful writes, and log each tool call so limit hits are diagnosable.
This commit is contained in:
@@ -4,6 +4,7 @@ export type AssistantChatMessage = {
|
||||
role: "user" | "assistant";
|
||||
content: string;
|
||||
imageUrl?: string;
|
||||
toolCalls?: Array<{ name: string; status: number }>;
|
||||
};
|
||||
|
||||
const STORAGE_VERSION = "v2";
|
||||
@@ -19,6 +20,14 @@ function isValidMessage(value: unknown): value is AssistantChatMessage {
|
||||
if (row.role !== "user" && row.role !== "assistant") return false;
|
||||
if (typeof row.content !== "string" || row.content.trim().length === 0) return false;
|
||||
if (row.imageUrl !== undefined && typeof row.imageUrl !== "string") return false;
|
||||
if (row.toolCalls !== undefined) {
|
||||
if (!Array.isArray(row.toolCalls)) return false;
|
||||
for (const call of row.toolCalls) {
|
||||
if (!call || typeof call !== "object") return false;
|
||||
const entry = call as Record<string, unknown>;
|
||||
if (typeof entry.name !== "string" || typeof entry.status !== "number") return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -229,7 +229,11 @@ export function AssistantPanel({ configured, userId, assistantName, assistantMod
|
||||
|
||||
setMessages((current) => [
|
||||
...current,
|
||||
{ role: "assistant", content: result.message.content },
|
||||
{
|
||||
role: "assistant",
|
||||
content: result.message.content,
|
||||
...(result.toolCalls.length > 0 ? { toolCalls: result.toolCalls } : {}),
|
||||
},
|
||||
]);
|
||||
scrollToBottom();
|
||||
} catch (err) {
|
||||
@@ -347,6 +351,13 @@ export function AssistantPanel({ configured, userId, assistantName, assistantMod
|
||||
/>
|
||||
) : null}
|
||||
{message.content}
|
||||
{message.role === "assistant" &&
|
||||
message.toolCalls &&
|
||||
message.toolCalls.length > 0 ? (
|
||||
<p className="mt-1 text-[10px] text-muted-foreground">
|
||||
{message.toolCalls.map((call) => `${call.name}→${call.status}`).join(" · ")}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
))}
|
||||
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
const MUTATION_TOOLS = new Set([
|
||||
"add_list_item",
|
||||
"update_list_item",
|
||||
"delete_list_item",
|
||||
"create_list",
|
||||
"update_list",
|
||||
"delete_list",
|
||||
"create_event",
|
||||
"update_event",
|
||||
"delete_event",
|
||||
"create_calendar",
|
||||
"create_note",
|
||||
"update_note",
|
||||
"delete_note",
|
||||
"create_journal_entry",
|
||||
"update_journal_entry",
|
||||
"delete_journal_entry",
|
||||
"create_garden_plant",
|
||||
"update_garden_plant",
|
||||
"delete_garden_plant",
|
||||
"create_garden_container",
|
||||
"update_garden_container",
|
||||
"delete_garden_container",
|
||||
"log_garden_care",
|
||||
"create_garden_care_schedule",
|
||||
"update_garden_care_schedule",
|
||||
"delete_garden_care_schedule",
|
||||
"schedule_garden_care_on_calendar",
|
||||
"create_bang",
|
||||
"update_bang",
|
||||
"delete_bang",
|
||||
"create_share_link",
|
||||
"revoke_share_link",
|
||||
]);
|
||||
|
||||
export function isMutationTool(name: string): boolean {
|
||||
return MUTATION_TOOLS.has(name);
|
||||
}
|
||||
|
||||
export function isSuccessfulWrite(name: string, status: number, argsJson = ""): boolean {
|
||||
if (!isSuccessfulStatus(status)) return false;
|
||||
if (isMutationTool(name)) return true;
|
||||
if (name !== "call_api") return false;
|
||||
try {
|
||||
const args = JSON.parse(argsJson || "{}") as { method?: string };
|
||||
const method = typeof args.method === "string" ? args.method.toUpperCase() : "";
|
||||
return method === "POST" || method === "PATCH" || method === "DELETE";
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function isSuccessfulStatus(status: number): boolean {
|
||||
return status >= 200 && status < 300;
|
||||
}
|
||||
|
||||
export function fingerprintToolCalls(calls: Array<{ name: string; arguments: string }>): string {
|
||||
return calls.map((call) => `${call.name}:${normalizeArgs(call.arguments)}`).join("|");
|
||||
}
|
||||
|
||||
export function truncateToolResult(result: string, maxChars = 6000): string {
|
||||
if (result.length <= maxChars) return result;
|
||||
return `${result.slice(0, maxChars)}\n…[truncated ${result.length - maxChars} chars]`;
|
||||
}
|
||||
|
||||
export function summarizeToolTrace(toolCalls: Array<{ name: string; status: number }>): string {
|
||||
if (toolCalls.length === 0) return "No tools were called.";
|
||||
return toolCalls.map((call) => `${call.name}→${call.status}`).join(", ");
|
||||
}
|
||||
|
||||
function normalizeArgs(argsJson: string): string {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(argsJson || "{}");
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
||||
return JSON.stringify(parsed);
|
||||
}
|
||||
const sorted: Record<string, unknown> = {};
|
||||
for (const key of Object.keys(parsed as Record<string, unknown>).sort()) {
|
||||
sorted[key] = (parsed as Record<string, unknown>)[key];
|
||||
}
|
||||
return JSON.stringify(sorted);
|
||||
} catch {
|
||||
return argsJson.trim();
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,16 @@
|
||||
import logger from "@/lib/logger";
|
||||
import { createLlmClient, type ChatMessage, type LlmClient } from "@/lib/llm";
|
||||
import { buildVisionContentParts, textFromMessageContent } from "@/lib/llm/content";
|
||||
import { AGENT_SYSTEM_PROMPT, AGENT_TOOLS, appendAgentRuntimeContext } from "../tools";
|
||||
import type { ClientChatMessage } from "../messages";
|
||||
import { describeToolActivity } from "../tool-labels";
|
||||
import { createApiToolExecutor, type ToolExecutor } from "../tool-executor";
|
||||
import {
|
||||
fingerprintToolCalls,
|
||||
isSuccessfulWrite,
|
||||
summarizeToolTrace,
|
||||
truncateToolResult,
|
||||
} from "./loop-guards";
|
||||
import { resolveAssistantImageDataUrls } from "./resolve-images";
|
||||
import { thinkingLabel, type AgentProgressEvent } from "./progress";
|
||||
|
||||
@@ -41,6 +48,21 @@ async function toLlmUserMessage(message: ClientChatMessage): Promise<ChatMessage
|
||||
};
|
||||
}
|
||||
|
||||
function forceReplyAfterWrite(writeNames: string[]): ChatMessage {
|
||||
return {
|
||||
role: "user",
|
||||
content: `The write already succeeded (${writeNames.join(", ")}). Stop calling tools and reply to the user in one short sentence confirming what you did.`,
|
||||
};
|
||||
}
|
||||
|
||||
function forceReplyAfterRepeat(): ChatMessage {
|
||||
return {
|
||||
role: "user",
|
||||
content:
|
||||
"You repeated the same tool call. Stop calling tools and reply with what you already know, or ask one clarifying question.",
|
||||
};
|
||||
}
|
||||
|
||||
export async function runAgentChat(options: {
|
||||
messages: ClientChatMessage[];
|
||||
request: Request;
|
||||
@@ -62,13 +84,16 @@ export async function runAgentChat(options: {
|
||||
const transcript: ChatMessage[] = [{ role: "system", content: systemPrompt }, ...userMessages];
|
||||
|
||||
const toolCalls: AgentToolCallSummary[] = [];
|
||||
const seenFingerprints = new Set<string>();
|
||||
const successfulWriteNames: string[] = [];
|
||||
let forceReply = false;
|
||||
|
||||
for (let round = 0; round < MAX_TOOL_ROUNDS; round += 1) {
|
||||
onProgress?.({ type: "thinking", label: thinkingLabel(round), round });
|
||||
|
||||
const completion = await llm.chatCompletion({
|
||||
messages: transcript,
|
||||
tools: AGENT_TOOLS,
|
||||
tools: forceReply ? undefined : AGENT_TOOLS,
|
||||
});
|
||||
|
||||
const assistantMessage = completion.message;
|
||||
@@ -76,6 +101,15 @@ export async function runAgentChat(options: {
|
||||
|
||||
if (!assistantMessage.tool_calls?.length) {
|
||||
onProgress?.({ type: "responding", label: "Writing a reply…" });
|
||||
logger.info(
|
||||
{
|
||||
msg: "agent.chat.done",
|
||||
rounds: round + 1,
|
||||
toolCalls: toolCalls.map((call) => `${call.name}:${call.status}`),
|
||||
forcedReply: forceReply,
|
||||
},
|
||||
"agent chat completed",
|
||||
);
|
||||
return {
|
||||
message: {
|
||||
role: "assistant",
|
||||
@@ -87,6 +121,36 @@ export async function runAgentChat(options: {
|
||||
};
|
||||
}
|
||||
|
||||
if (forceReply) {
|
||||
logger.warn(
|
||||
{
|
||||
msg: "agent.chat.forced_tools_ignored",
|
||||
round,
|
||||
names: assistantMessage.tool_calls.map((c) => c.function.name),
|
||||
},
|
||||
"model kept calling tools after force-reply",
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
const fingerprint = fingerprintToolCalls(
|
||||
assistantMessage.tool_calls.map((call) => ({
|
||||
name: call.function.name,
|
||||
arguments: call.function.arguments,
|
||||
})),
|
||||
);
|
||||
|
||||
if (seenFingerprints.has(fingerprint)) {
|
||||
logger.warn(
|
||||
{ msg: "agent.chat.duplicate_tools", round, fingerprint, toolCalls: fingerprint },
|
||||
"duplicate tool round detected",
|
||||
);
|
||||
transcript.push(forceReplyAfterRepeat());
|
||||
forceReply = true;
|
||||
continue;
|
||||
}
|
||||
seenFingerprints.add(fingerprint);
|
||||
|
||||
for (const toolCall of assistantMessage.tool_calls) {
|
||||
const label = describeToolActivity(toolCall.function.name, toolCall.function.arguments);
|
||||
onProgress?.({ type: "tool", name: toolCall.function.name, label });
|
||||
@@ -107,21 +171,69 @@ export async function runAgentChat(options: {
|
||||
}
|
||||
|
||||
toolCalls.push({ name: toolCall.function.name, status });
|
||||
logger.info(
|
||||
{
|
||||
msg: "agent.chat.tool",
|
||||
round,
|
||||
name: toolCall.function.name,
|
||||
status,
|
||||
args: toolCall.function.arguments.slice(0, 300),
|
||||
},
|
||||
"agent tool call",
|
||||
);
|
||||
|
||||
if (isSuccessfulWrite(toolCall.function.name, status, toolCall.function.arguments)) {
|
||||
successfulWriteNames.push(toolCall.function.name);
|
||||
}
|
||||
|
||||
transcript.push({
|
||||
role: "tool",
|
||||
tool_call_id: toolCall.id,
|
||||
name: toolCall.function.name,
|
||||
content: result,
|
||||
content: truncateToolResult(result),
|
||||
});
|
||||
}
|
||||
|
||||
if (successfulWriteNames.length > 0) {
|
||||
transcript.push(forceReplyAfterWrite(successfulWriteNames));
|
||||
forceReply = true;
|
||||
}
|
||||
}
|
||||
|
||||
onProgress?.({ type: "responding", label: "Wrapping up…" });
|
||||
const trace = summarizeToolTrace(toolCalls);
|
||||
|
||||
if (successfulWriteNames.length > 0) {
|
||||
logger.warn(
|
||||
{
|
||||
msg: "agent.chat.forced_summary",
|
||||
rounds: MAX_TOOL_ROUNDS,
|
||||
toolCalls: toolCalls.map((call) => `${call.name}:${call.status}`),
|
||||
},
|
||||
"agent summarizing after write without clean stop",
|
||||
);
|
||||
return {
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: `Done. Completed: ${[...new Set(successfulWriteNames)].join(", ")}.`,
|
||||
},
|
||||
toolCalls,
|
||||
};
|
||||
}
|
||||
|
||||
logger.warn(
|
||||
{
|
||||
msg: "agent.chat.tool_limit",
|
||||
rounds: MAX_TOOL_ROUNDS,
|
||||
toolCalls: toolCalls.map((call) => `${call.name}:${call.status}`),
|
||||
forcedReply: forceReply,
|
||||
},
|
||||
"agent hit tool-call limit",
|
||||
);
|
||||
return {
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: "I hit the tool-call limit for this request. Please try a simpler question.",
|
||||
content: `I hit the tool-call limit for this request. Tools used: ${trace}.`,
|
||||
},
|
||||
toolCalls,
|
||||
};
|
||||
|
||||
@@ -451,13 +451,18 @@ async function getApiDocs(args: Record<string, unknown>): Promise<ApiCallResult>
|
||||
const specPath = path.join(process.cwd(), "docs", "api", "openapi.yaml");
|
||||
const spec = await readFile(specPath, "utf8");
|
||||
const search = typeof args.search === "string" ? args.search.trim().toLowerCase() : "";
|
||||
const pathLines = spec
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.startsWith("/api/v1/"));
|
||||
|
||||
if (!search) {
|
||||
return {
|
||||
status: 200,
|
||||
body: {
|
||||
spec,
|
||||
hint: "Pass search to filter paths, or use call_api with a /api/v1/* path.",
|
||||
paths: pathLines.slice(0, 80),
|
||||
pathCount: pathLines.length,
|
||||
hint: "Pass search (e.g. calendar, events, lists) to get matching lines. Do not request the full OpenAPI dump.",
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -469,8 +474,8 @@ async function getApiDocs(args: Record<string, unknown>): Promise<ApiCallResult>
|
||||
body: {
|
||||
search,
|
||||
matchCount: matches.length,
|
||||
matches: matches.slice(0, 100),
|
||||
hint: "Use call_api with method and path from the matches above.",
|
||||
matches: matches.slice(0, 40),
|
||||
hint: "Use a dedicated tool when one exists; otherwise call_api with method and path from the matches above.",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -718,15 +718,16 @@ export const AGENT_TOOLS: AgentToolDefinition[] = [
|
||||
function: {
|
||||
name: "get_api_docs",
|
||||
description:
|
||||
"Read famapp REST API documentation (OpenAPI). Use when unsure which endpoint to call or no dedicated tool exists. Pass search to filter relevant paths.",
|
||||
"Search famapp REST API docs for /api/v1 paths. Always pass search. Returns matching lines only — not the full OpenAPI file.",
|
||||
parameters: {
|
||||
type: "object",
|
||||
properties: {
|
||||
search: {
|
||||
type: "string",
|
||||
description: "Optional keyword to filter paths (e.g. garden, share, journal)",
|
||||
description: "Keyword to filter paths (e.g. garden, share, journal, events)",
|
||||
},
|
||||
},
|
||||
required: ["search"],
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -767,10 +768,11 @@ export const AGENT_SYSTEM_PROMPT = `You are the famapp household assistant. Help
|
||||
|
||||
Use the provided tools to read and update data. Prefer calling tools instead of guessing. Be concise and friendly.
|
||||
|
||||
Prefer a dedicated tool when one exists (create_event, add_list_item, create_note, etc.). When no dedicated tool fits, or you need an endpoint that is not wrapped yet:
|
||||
1. Call get_api_docs with a relevant search term to find the right /api/v1/* endpoint.
|
||||
Prefer a dedicated tool when one exists (create_event, add_list_item, create_note, etc.). After a successful write (2xx), stop calling tools and confirm in one short sentence — do not re-list or re-create.
|
||||
When no dedicated tool fits, or you need an endpoint that is not wrapped yet:
|
||||
1. Call get_api_docs with a relevant search term (always pass search; never request the full spec).
|
||||
2. Call call_api with the documented method, path, query, and body.
|
||||
If a tool call fails, read the error body and fix the arguments before trying a different approach.
|
||||
If a tool call fails, read the error body and fix the arguments before trying a different approach. Do not repeat the exact same tool call.
|
||||
|
||||
When the user sends a photo, read dates, times, locations, and action items from it, then use tools to act.
|
||||
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import {
|
||||
fingerprintToolCalls,
|
||||
isSuccessfulWrite,
|
||||
summarizeToolTrace,
|
||||
truncateToolResult,
|
||||
} from "../../src/modules/agent/server/loop-guards";
|
||||
import { runAgentChat } from "../../src/modules/agent/server/run";
|
||||
import type {
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResult,
|
||||
LlmClient,
|
||||
} from "../../src/lib/llm/types";
|
||||
|
||||
describe("loop-guards", () => {
|
||||
it("fingerprints tool calls stably regardless of key order", () => {
|
||||
const a = fingerprintToolCalls([
|
||||
{ name: "create_event", arguments: '{"title":"Dentist","calendarName":"Family"}' },
|
||||
]);
|
||||
const b = fingerprintToolCalls([
|
||||
{ name: "create_event", arguments: '{"calendarName":"Family","title":"Dentist"}' },
|
||||
]);
|
||||
assert.equal(a, b);
|
||||
});
|
||||
|
||||
it("treats create_event 201 as a successful write", () => {
|
||||
assert.equal(isSuccessfulWrite("create_event", 201), true);
|
||||
assert.equal(isSuccessfulWrite("list_calendars", 200), false);
|
||||
assert.equal(
|
||||
isSuccessfulWrite("call_api", 201, '{"method":"POST","path":"/api/v1/events"}'),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
isSuccessfulWrite("call_api", 200, '{"method":"GET","path":"/api/v1/events"}'),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("truncates oversized tool results", () => {
|
||||
const result = truncateToolResult("x".repeat(7000), 100);
|
||||
assert.ok(result.length < 200);
|
||||
assert.match(result, /truncated/);
|
||||
});
|
||||
|
||||
it("summarizes tool traces", () => {
|
||||
assert.equal(
|
||||
summarizeToolTrace([
|
||||
{ name: "list_calendars", status: 200 },
|
||||
{ name: "create_event", status: 201 },
|
||||
]),
|
||||
"list_calendars→200, create_event→201",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("runAgentChat loop guards", () => {
|
||||
it("stops after a successful write instead of looping", async () => {
|
||||
let calls = 0;
|
||||
const llm: LlmClient = {
|
||||
async chatCompletion(request: ChatCompletionRequest): Promise<ChatCompletionResult> {
|
||||
calls += 1;
|
||||
if (calls === 1) {
|
||||
return {
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: "1",
|
||||
type: "function",
|
||||
function: {
|
||||
name: "create_event",
|
||||
arguments: JSON.stringify({
|
||||
title: "Dentist",
|
||||
startAt: "2026-07-10T15:00:00.000Z",
|
||||
endAt: "2026-07-10T16:00:00.000Z",
|
||||
}),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
finishReason: "tool_calls",
|
||||
};
|
||||
}
|
||||
|
||||
assert.equal(request.tools, undefined);
|
||||
return {
|
||||
message: { role: "assistant", content: "Added Dentist to your calendar." },
|
||||
finishReason: "stop",
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
const result = await runAgentChat({
|
||||
messages: [{ role: "user", content: "add dentist tomorrow at 10" }],
|
||||
request: new Request("http://localhost:3000/api/agent/chat"),
|
||||
llm,
|
||||
executeTool: async () =>
|
||||
JSON.stringify({ status: 201, body: { id: "evt-1", title: "Dentist" } }),
|
||||
});
|
||||
|
||||
assert.equal(calls, 2);
|
||||
assert.equal(result.toolCalls.length, 1);
|
||||
assert.equal(result.toolCalls[0]?.name, "create_event");
|
||||
assert.match(result.message.content, /Dentist/);
|
||||
});
|
||||
|
||||
it("breaks duplicate identical tool rounds", async () => {
|
||||
let calls = 0;
|
||||
const llm: LlmClient = {
|
||||
async chatCompletion(): Promise<ChatCompletionResult> {
|
||||
calls += 1;
|
||||
if (calls <= 2) {
|
||||
return {
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: String(calls),
|
||||
type: "function",
|
||||
function: {
|
||||
name: "list_calendars",
|
||||
arguments: "{}",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
finishReason: "tool_calls",
|
||||
};
|
||||
}
|
||||
return {
|
||||
message: { role: "assistant", content: "You have one Family calendar." },
|
||||
finishReason: "stop",
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
const result = await runAgentChat({
|
||||
messages: [{ role: "user", content: "what calendars do I have?" }],
|
||||
request: new Request("http://localhost:3000/api/agent/chat"),
|
||||
llm,
|
||||
executeTool: async () =>
|
||||
JSON.stringify({ status: 200, body: [{ id: "cal-1", name: "Family" }] }),
|
||||
});
|
||||
|
||||
assert.equal(calls, 3);
|
||||
assert.equal(result.toolCalls.length, 1);
|
||||
assert.match(result.message.content, /Family/);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user