famapp/tests/unit/llm-models.test.ts

import assert from "node:assert/strict";
import { describe, it } from "node:test";
import {
  isValidLlmModelId,
  listLlmModels,
  normalizeLlmModelsPayload,
  resolveAssistantModel,
} from "../../src/lib/llm/models";
import type { LlmConfig } from "../../src/lib/llm/config";

const openAiConfig: LlmConfig = {
  provider: "openai",
  baseUrl: "https://llm.example.test/v1",
  apiKey: "secret",
  model: "llama3.2",
};

describe("normalizeLlmModelsPayload", () => {
  it("normalizes OpenAI-compatible data arrays", () => {
    const models = normalizeLlmModelsPayload({
      data: [{ id: "qwen2.5-coder" }, { id: "llama3.2" }, { id: "qwen2.5-coder" }],
    });

    assert.deepEqual(models, [
      { id: "llama3.2", label: "llama3.2" },
      { id: "qwen2.5-coder", label: "qwen2.5-coder" },
    ]);
  });

  it("ignores invalid or empty model rows", () => {
    const models = normalizeLlmModelsPayload({
      data: [
        { id: "" },
        { id: "   " },
        { id: "bad model" },
        { id: " llama3.2 " },
        { object: "model" },
      ],
    });

    assert.deepEqual(models, []);
  });
});

describe("isValidLlmModelId", () => {
  it("accepts common provider model IDs", () => {
    assert.equal(isValidLlmModelId("llama3.2"), true);
    assert.equal(isValidLlmModelId("qwen2.5-coder:latest"), true);
    assert.equal(isValidLlmModelId("hf.co/ginnoir/model-v1"), true);
  });

  it("rejects empty, whitespace, and overlong model IDs", () => {
    assert.equal(isValidLlmModelId(""), false);
    assert.equal(isValidLlmModelId("bad model"), false);
    assert.equal(isValidLlmModelId("x".repeat(129)), false);
  });
});

describe("listLlmModels", () => {
  it("fetches provider models with API key auth and includes the fallback model", async () => {
    const requests: Request[] = [];
    const result = await listLlmModels({
      config: openAiConfig,
      fetchImpl: async (input, init) => {
        requests.push(new Request(input, init));
        return Response.json({ data: [{ id: "qwen2.5-coder" }] });
      },
    });

    assert.equal(requests[0]?.url, "https://llm.example.test/v1/models");
    assert.equal(requests[0]?.headers.get("authorization"), "Bearer secret");
    assert.deepEqual(result.models, [
      { id: "llama3.2", label: "llama3.2" },
      { id: "qwen2.5-coder", label: "qwen2.5-coder" },
    ]);
    assert.equal(result.fallbackModel, "llama3.2");
    assert.equal(result.degraded, false);
  });

  it("falls back to LLM_MODEL when provider discovery fails", async () => {
    const result = await listLlmModels({
      config: openAiConfig,
      fetchImpl: async () => new Response("nope", { status: 500 }),
    });

    assert.deepEqual(result.models, [{ id: "llama3.2", label: "llama3.2" }]);
    assert.equal(result.fallbackModel, "llama3.2");
    assert.equal(result.degraded, true);
  });

  it("uses fallback only for mock provider config", async () => {
    const result = await listLlmModels({
      config: { provider: "mock", baseUrl: null, apiKey: null, model: "llama3.2" },
      fetchImpl: async () => {
        throw new Error("fetch should not run for mock config");
      },
    });

    assert.deepEqual(result.models, [{ id: "llama3.2", label: "llama3.2" }]);
    assert.equal(result.degraded, false);
  });
});

describe("resolveAssistantModel", () => {
  it("uses a valid requested model before saved and fallback values", () => {
    const resolved = resolveAssistantModel({
      requestedModel: "qwen2.5-coder",
      savedModel: "llama3.2",
      fallbackModel: "llama3.2",
      models: [
        { id: "llama3.2", label: "llama3.2" },
        { id: "qwen2.5-coder", label: "qwen2.5-coder" },
      ],
    });

    assert.deepEqual(resolved, { ok: true, model: "qwen2.5-coder" });
  });

  it("rejects invalid requested models", () => {
    const resolved = resolveAssistantModel({
      requestedModel: "bad model",
      savedModel: null,
      fallbackModel: "llama3.2",
      models: [
        { id: "llama3.2", label: "llama3.2" },
        { id: "bad model", label: "bad model" },
      ],
    });

    assert.deepEqual(resolved, {
      ok: false,
      model: "llama3.2",
      error: "Invalid assistant model",
    });
  });

  it("rejects empty requested models", () => {
    const resolved = resolveAssistantModel({
      requestedModel: "",
      savedModel: null,
      fallbackModel: "llama3.2",
      models: [{ id: "llama3.2", label: "llama3.2" }],
    });

    assert.deepEqual(resolved, {
      ok: false,
      model: "llama3.2",
      error: "Invalid assistant model",
    });
  });

  it("rejects unavailable requested models", () => {
    const resolved = resolveAssistantModel({
      requestedModel: "missing",
      savedModel: null,
      fallbackModel: "llama3.2",
      models: [{ id: "llama3.2", label: "llama3.2" }],
    });

    assert.deepEqual(resolved, {
      ok: false,
      model: "llama3.2",
      error: "Invalid assistant model",
    });
  });

  it("silently falls back when a saved model is gone", () => {
    const resolved = resolveAssistantModel({
      requestedModel: null,
      savedModel: "old-model",
      fallbackModel: "llama3.2",
      models: [{ id: "llama3.2", label: "llama3.2" }],
    });

    assert.deepEqual(resolved, { ok: true, model: "llama3.2" });
  });
});