openAgent

1import { describe, expect, test } from "bun:test"
2import { Effect } from "effect"
3import { CacheHint, LLM, Message } from "../src"
4import { Auth, LLMClient } from "../src/route"
5import { AmazonBedrock } from "../src/providers"
6import * as AnthropicMessages from "../src/protocols/anthropic-messages"
7import * as Gemini from "../src/protocols/gemini"
8import * as OpenAIChat from "../src/protocols/openai-chat"
9import { applyCachePolicy } from "../src/cache-policy"
10import { it } from "./lib/effect"
11 
12const anthropicModel = AnthropicMessages.route
13  .with({ endpoint: { baseURL: "https://api.anthropic.test/v1/" }, auth: Auth.header("x-api-key", "test") })
14  .model({ id: "claude-sonnet-4-5" })
15 
16const bedrockModel = AmazonBedrock.configure({
17  credentials: { region: "us-east-1", accessKeyId: "fixture", secretAccessKey: "fixture" },
18}).model("anthropic.claude-3-5-sonnet-20241022-v2:0")
19 
20const openaiModel = OpenAIChat.route
21  .with({ endpoint: { baseURL: "https://api.openai.test/v1/" }, auth: Auth.bearer("test") })
22  .model({ id: "gpt-4o-mini" })
23 
24const geminiModel = Gemini.route
25  .with({
26    endpoint: { baseURL: "https://generativelanguage.test/v1beta/" },
27    auth: Auth.header("x-goog-api-key", "test"),
28  })
29  .model({ id: "gemini-2.5-flash" })
30 
31describe("applyCachePolicy", () => {
32  it.effect("undefined cache resolves to 'auto' (the recommended default)", () =>
33    Effect.gen(function* () {
34      const prepared = yield* LLMClient.prepare(
35        LLM.request({
36          model: anthropicModel,
37          system: "You are concise.",
38          prompt: "hi",
39        }),
40      )
41 
42      // No explicit cache field → auto policy fires → last system part + latest
43      // user message both get cache_control markers.
44      expect(prepared.body).toMatchObject({
45        system: [{ type: "text", text: "You are concise.", cache_control: { type: "ephemeral" } }],
46        messages: [{ role: "user", content: [{ type: "text", text: "hi", cache_control: { type: "ephemeral" } }] }],
47      })
48    }),
49  )
50 
51  it.effect("'auto' marks the last tool, last system part, and latest user message on Anthropic", () =>
52    Effect.gen(function* () {
53      const prepared = yield* LLMClient.prepare(
54        LLM.request({
55          model: anthropicModel,
56          system: "Sys A",
57          tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
58          messages: [
59            Message.user("first user"),
60            Message.assistant("assistant reply"),
61            Message.user("latest user message"),
62          ],
63          cache: "auto",
64        }),
65      )
66 
67      expect(prepared.body).toMatchObject({
68        tools: [{ name: "t1", cache_control: { type: "ephemeral" } }],
69        system: [{ type: "text", text: "Sys A", cache_control: { type: "ephemeral" } }],
70        messages: [
71          { role: "user", content: [{ type: "text", text: "first user" }] },
72          { role: "assistant", content: [{ type: "text", text: "assistant reply" }] },
73          {
74            role: "user",
75            content: [{ type: "text", text: "latest user message", cache_control: { type: "ephemeral" } }],
76          },
77        ],
78      })
79    }),
80  )
81 
82  it.effect("'auto' is a no-op on OpenAI (implicit caching protocol)", () =>
83    Effect.gen(function* () {
84      const prepared = yield* LLMClient.prepare(
85        LLM.request({
86          model: openaiModel,
87          system: "Sys",
88          prompt: "hi",
89          cache: "auto",
90        }),
91      )
92 
93      const body = prepared.body as { messages: Array<{ content: unknown }> }
94      // OpenAI doesn't accept cache_control on messages — policy must skip.
95      const flat = JSON.stringify(body)
96      expect(flat).not.toContain("cache_control")
97      expect(flat).not.toContain("cachePoint")
98    }),
99  )
100 
101  it.effect("'auto' is a no-op on Gemini (out-of-band caching protocol)", () =>
102    Effect.gen(function* () {
103      const prepared = yield* LLMClient.prepare(
104        LLM.request({
105          model: geminiModel,
106          system: "Sys",
107          prompt: "hi",
108          cache: "auto",
109        }),
110      )
111 
112      const flat = JSON.stringify(prepared.body)
113      expect(flat).not.toContain("cache_control")
114      expect(flat).not.toContain("cachePoint")
115    }),
116  )
117 
118  it.effect("'auto' on Bedrock emits cachePoint markers in the right places", () =>
119    Effect.gen(function* () {
120      const prepared = yield* LLMClient.prepare(
121        LLM.request({
122          model: bedrockModel,
123          system: "Sys",
124          tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
125          messages: [Message.user("first user"), Message.assistant("reply"), Message.user("latest user")],
126          cache: "auto",
127        }),
128      )
129 
130      expect(prepared.body).toMatchObject({
131        toolConfig: {
132          tools: [{ toolSpec: { name: "t1" } }, { cachePoint: { type: "default" } }],
133        },
134        system: [{ text: "Sys" }, { cachePoint: { type: "default" } }],
135        messages: [
136          { role: "user", content: [{ text: "first user" }] },
137          { role: "assistant", content: [{ text: "reply" }] },
138          { role: "user", content: [{ text: "latest user" }, { cachePoint: { type: "default" } }] },
139        ],
140      })
141    }),
142  )
143 
144  it.effect("'none' disables auto placement even when manual hints exist", () =>
145    Effect.gen(function* () {
146      const prepared = yield* LLMClient.prepare(
147        LLM.request({
148          model: anthropicModel,
149          system: "Sys",
150          tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
151          prompt: "hi",
152          cache: "none",
153        }),
154      )
155 
156      expect(prepared.body).toMatchObject({
157        tools: [{ name: "t1", cache_control: undefined }],
158        system: [{ type: "text", text: "Sys", cache_control: undefined }],
159      })
160    }),
161  )
162 
163  it.effect("granular object form: tools-only marks just tools", () =>
164    Effect.gen(function* () {
165      const prepared = yield* LLMClient.prepare(
166        LLM.request({
167          model: anthropicModel,
168          system: "Sys",
169          tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
170          prompt: "hi",
171          cache: { tools: true },
172        }),
173      )
174 
175      expect(prepared.body).toMatchObject({
176        tools: [{ name: "t1", cache_control: { type: "ephemeral" } }],
177        system: [{ type: "text", text: "Sys", cache_control: undefined }],
178      })
179    }),
180  )
181 
182  it.effect("auto policy preserves manual CacheHints on other parts", () =>
183    Effect.gen(function* () {
184      const prepared = yield* LLMClient.prepare(
185        LLM.request({
186          model: anthropicModel,
187          system: [
188            { type: "text", text: "first system", cache: new CacheHint({ type: "ephemeral", ttlSeconds: 3600 }) },
189            { type: "text", text: "last system" },
190          ],
191          prompt: "hi",
192          cache: "auto",
193        }),
194      )
195 
196      const body = prepared.body as { system: Array<{ text: string; cache_control?: unknown }> }
197      expect(body.system[0]?.cache_control).toEqual({ type: "ephemeral", ttl: "1h" })
198      expect(body.system[1]?.cache_control).toEqual({ type: "ephemeral" })
199    }),
200  )
201 
202  it.effect("ttlSeconds in the policy flows through to wire markers", () =>
203    Effect.gen(function* () {
204      const prepared = yield* LLMClient.prepare(
205        LLM.request({
206          model: anthropicModel,
207          system: "Sys",
208          prompt: "hi",
209          cache: { system: true, ttlSeconds: 3600 },
210        }),
211      )
212 
213      expect(prepared.body).toMatchObject({
214        system: [{ type: "text", text: "Sys", cache_control: { type: "ephemeral", ttl: "1h" } }],
215      })
216    }),
217  )
218 
219  it.effect("messages: { tail: 2 } marks the last 2 message boundaries", () =>
220    Effect.gen(function* () {
221      const prepared = yield* LLMClient.prepare(
222        LLM.request({
223          model: anthropicModel,
224          messages: [Message.user("u1"), Message.assistant("a1"), Message.user("u2"), Message.assistant("a2")],
225          cache: { messages: { tail: 2 } },
226        }),
227      )
228 
229      const body = prepared.body as { messages: Array<{ content: Array<{ cache_control?: unknown }> }> }
230      expect(body.messages[0]?.content[0]?.cache_control).toBeUndefined()
231      expect(body.messages[1]?.content[0]?.cache_control).toBeUndefined()
232      expect(body.messages[2]?.content[0]?.cache_control).toEqual({ type: "ephemeral" })
233      expect(body.messages[3]?.content[0]?.cache_control).toEqual({ type: "ephemeral" })
234    }),
235  )
236 
237  it.effect("'latest-assistant' marks the last assistant message", () =>
238    Effect.gen(function* () {
239      const prepared = yield* LLMClient.prepare(
240        LLM.request({
241          model: anthropicModel,
242          messages: [Message.user("u1"), Message.assistant("a1"), Message.user("u2")],
243          cache: { messages: "latest-assistant" },
244        }),
245      )
246 
247      const body = prepared.body as { messages: Array<{ content: Array<{ cache_control?: unknown }> }> }
248      expect(body.messages[0]?.content[0]?.cache_control).toBeUndefined()
249      expect(body.messages[1]?.content[0]?.cache_control).toEqual({ type: "ephemeral" })
250      expect(body.messages[2]?.content[0]?.cache_control).toBeUndefined()
251    }),
252  )
253 
254  test("returns the same request reference when policy is a no-op (pure function)", () => {
255    const request = LLM.request({
256      model: anthropicModel,
257      prompt: "hi",
258      cache: "none",
259    })
260    expect(applyCachePolicy(request)).toBe(request)
261  })
262})
263