openAgent

1import { describe, expect, test } from "bun:test"
2import { Effect, Schema } from "effect"
3import { CodeMode, Tool, toolError } from "../src/index.js"
4 
5// Wave 5 acceptance suite: first-class promise values. Un-awaited tool calls start eagerly on
6// supervised fibers, `await` settles them, and Promise.all/allSettled/race/resolve/reject are
7// ordinary functions over arbitrary arrays mixing promises and plain values.
8 
9type Trace = {
10  starts: Array<number>
11  active: number
12  maxActive: number
13  completed: number
14  interrupted: number
15}
16 
17const makeTrace = (): Trace => ({ starts: [], active: 0, maxActive: 0, completed: 0, interrupted: 0 })
18 
19/** Echoes `id` after `ms` milliseconds, recording start order, live concurrency, and interruption. */
20const sleepyTool = (trace: Trace) =>
21  Tool.make({
22    description: "Echo an id after a delay",
23    input: Schema.Struct({ id: Schema.Number, ms: Schema.optionalKey(Schema.Number) }),
24    output: Schema.Number,
25    run: ({ id, ms }) =>
26      Effect.gen(function* () {
27        trace.starts.push(id)
28        trace.active += 1
29        trace.maxActive = Math.max(trace.maxActive, trace.active)
30        yield* Effect.sleep(ms ?? 20)
31        trace.active -= 1
32        trace.completed += 1
33        return id
34      }).pipe(
35        Effect.onInterrupt(() =>
36          Effect.sync(() => {
37            trace.active -= 1
38            trace.interrupted += 1
39          }),
40        ),
41      ),
42  })
43 
44const failingTool = Tool.make({
45  description: "Always refuse",
46  input: Schema.Struct({}),
47  output: Schema.String,
48  run: () => Effect.fail(toolError("Lookup refused")),
49})
50 
51const run = (
52  code: string,
53  options: { trace?: Trace; limits?: CodeMode.ExecutionLimits } = {},
54): Promise<CodeMode.Result> => {
55  const trace = options.trace ?? makeTrace()
56  return Effect.runPromise(
57    CodeMode.execute({
58      tools: { host: { sleepy: sleepyTool(trace), fail: failingTool } },
59      code,
60      ...(options.limits ? { limits: options.limits } : {}),
61    }),
62  )
63}
64 
65const value = async (code: string, options: { trace?: Trace; limits?: CodeMode.ExecutionLimits } = {}) => {
66  const result = await run(code, options)
67  if (!result.ok) throw new Error(`expected success, got ${result.error.kind}: ${result.error.message}`)
68  return result.value
69}
70 
71const error = async (code: string, options: { trace?: Trace; limits?: CodeMode.ExecutionLimits } = {}) => {
72  const result = await run(code, options)
73  if (result.ok) throw new Error(`expected failure, got value ${JSON.stringify(result.value)}`)
74  return result.error
75}
76 
77describe("first-class promise values", () => {
78  test("an un-awaited tool call starts eagerly, in call order, before any await", async () => {
79    const trace = makeTrace()
80    const result = await value(
81      `
82        const a = tools.host.sleepy({ id: 1, ms: 40 })
83        const b = tools.host.sleepy({ id: 2, ms: 40 })
84        const rb = await b
85        const ra = await a
86        return [ra, rb]
87      `,
88      { trace },
89    )
90    expect(result).toEqual([1, 2])
91    expect(trace.starts).toEqual([1, 2])
92    // Both calls overlapped even though they were awaited sequentially.
93    expect(trace.maxActive).toBeGreaterThan(1)
94  })
95 
96  test("awaiting the same promise twice settles once and never re-runs the call", async () => {
97    const result = await run(`
98      const p = tools.host.sleepy({ id: 7 })
99      const x = await p
100      const y = await p
101      return [x, y]
102    `)
103    expect(result.ok).toBe(true)
104    if (!result.ok) return
105    expect(result.value).toEqual([7, 7])
106    expect(result.toolCalls).toStrictEqual([{ name: "host.sleepy" }])
107  })
108 
109  test("await of a non-promise value is a passthrough no-op", async () => {
110    expect(await value(`return await 42`)).toBe(42)
111    expect(await value(`const x = await "s"; return x`)).toBe("s")
112    expect(await value(`return await null`)).toBeNull()
113    expect(await value(`return (await [1, 2]).length`)).toBe(2)
114  })
115 
116  test("returning an un-awaited tool call resolves it (async-function return semantics)", async () => {
117    expect(await value(`return tools.host.sleepy({ id: 9 })`)).toBe(9)
118  })
119 
120  test("typeof a promise is 'object', and console.log renders it sensibly", async () => {
121    const result = await run(`
122      const p = Promise.resolve(1)
123      console.log(p)
124      return typeof p
125    `)
126    expect(result.ok).toBe(true)
127    if (!result.ok) return
128    expect(result.value).toBe("object")
129    expect(result.logs).toStrictEqual(["[Promise (await it to get its value)]"])
130  })
131 
132  test("an awaited failure is catchable exactly like a synchronous throw", async () => {
133    expect(
134      await value(`
135      const p = tools.host.fail({})
136      try {
137        await p
138        return "no"
139      } catch (e) {
140        return e.message
141      }
142    `),
143    ).toBe("Lookup refused")
144  })
145 
146  test("a fire-and-forget call completes before the execution ends", async () => {
147    const trace = makeTrace()
148    const result = await value(
149      `
150        tools.host.sleepy({ id: 1, ms: 30 })
151        return "done"
152      `,
153      { trace },
154    )
155    expect(result).toBe("done")
156    expect(trace.completed).toBe(1)
157    expect(trace.interrupted).toBe(0)
158  })
159 
160  test("a never-awaited failing call surfaces as an unhandled-rejection diagnostic", async () => {
161    const diagnostic = await error(`
162      tools.host.fail({})
163      return "done"
164    `)
165    expect(diagnostic.kind).toBe("ToolFailure")
166    expect(diagnostic.message).toContain("Unhandled rejection from an un-awaited tool call")
167    expect(diagnostic.message).toContain("Lookup refused")
168    expect(diagnostic.suggestions?.join(" ")).toContain("await tools.ns.tool(...)")
169  })
170})
171 
172describe("promises at data boundaries", () => {
173  test("returning an un-awaited promise inside data is a clear await-hinting diagnostic", async () => {
174    const diagnostic = await error(`return { result: tools.host.sleepy({ id: 1 }) }`)
175    expect(diagnostic.kind).toBe("InvalidDataValue")
176    expect(diagnostic.message).toContain("un-awaited Promise")
177    expect(diagnostic.message).toContain("await tools.ns.tool(...)")
178  })
179 
180  test("passing an un-awaited promise as a tool argument is a clear diagnostic", async () => {
181    const diagnostic = await error(`return await tools.host.sleepy({ id: tools.host.sleepy({ id: 1 }) })`)
182    expect(diagnostic.kind).toBe("InvalidDataValue")
183    expect(diagnostic.message).toContain("un-awaited Promise")
184  })
185 
186  test("JSON.stringify of a promise is a diagnostic, not '{}'", async () => {
187    const diagnostic = await error(`return JSON.stringify(Promise.resolve(1))`)
188    expect(diagnostic.kind).toBe("InvalidDataValue")
189    expect(diagnostic.message).toContain("un-awaited Promise")
190  })
191 
192  test("operators reject promise operands", async () => {
193    const diagnostic = await error(`return Promise.resolve(1) + 1`)
194    expect(diagnostic.kind).toBe("InvalidDataValue")
195  })
196})
197 
198describe("Promise.all over arbitrary arrays", () => {
199  test("mixes promises and plain values, preserving order", async () => {
200    expect(
201      await value(`
202      return await Promise.all([tools.host.sleepy({ id: 1 }), "plain", tools.host.sleepy({ id: 2 }), 42])
203    `),
204    ).toEqual([1, "plain", 2, 42])
205  })
206 
207  test("accepts arrays built beforehand, passed as identifiers, and spread elements", async () => {
208    expect(
209      await value(`
210      const calls = []
211      calls.push(tools.host.sleepy({ id: 1 }))
212      calls.push(7)
213      const more = [tools.host.sleepy({ id: 2 })]
214      const batch = [...calls, ...more, "x"]
215      return await Promise.all(batch)
216    `),
217    ).toEqual([1, 7, 2, "x"])
218  })
219 
220  test("runs items.map tool calls in parallel", async () => {
221    const trace = makeTrace()
222    const result = await value(
223      `
224        const ids = [1, 2, 3, 4]
225        return await Promise.all(ids.map((id) => tools.host.sleepy({ id, ms: 40 })))
226      `,
227      { trace },
228    )
229    expect(result).toEqual([1, 2, 3, 4])
230    // maxActive counts truly-overlapping live executions, so > 1 proves real
231    // parallelism deterministically - no wall-clock assertion needed.
232    expect(trace.maxActive).toBeGreaterThan(1)
233  })
234 
235  test("caps live tool-call concurrency at the fixed internal constant (8)", async () => {
236    const trace = makeTrace()
237    const result = await value(
238      `
239        const ids = []
240        for (let i = 0; i < 20; i += 1) ids.push(i)
241        const results = await Promise.all(ids.map((id) => tools.host.sleepy({ id, ms: 10 })))
242        return results.length
243      `,
244      { trace },
245    )
246    expect(result).toBe(20)
247    expect(trace.maxActive).toBeGreaterThan(1)
248    expect(trace.maxActive).toBeLessThanOrEqual(8)
249  })
250 
251  test("resolves the empty array", async () => {
252    expect(await value(`return await Promise.all([])`)).toEqual([])
253  })
254 
255  test("rejects with the first failure, catchable in-program", async () => {
256    expect(
257      await value(`
258      try {
259        await Promise.all([tools.host.sleepy({ id: 1 }), tools.host.fail({})])
260        return "no"
261      } catch (e) {
262        return e.message
263      }
264    `),
265    ).toBe("Lookup refused")
266  })
267 
268  test("a non-collection argument is a clear error", async () => {
269    const diagnostic = await error(`return await Promise.all(42)`)
270    expect(diagnostic.message).toContain("Promise.all expects an array")
271  })
272 
273  test("exceeding maxToolCalls inside Promise.all is a ToolCallLimitExceeded diagnostic", async () => {
274    const diagnostic = await error(
275      `return await Promise.all([tools.host.sleepy({ id: 1 }), tools.host.sleepy({ id: 2 }), tools.host.sleepy({ id: 3 })])`,
276      { limits: { maxToolCalls: 2 } },
277    )
278    expect(diagnostic.kind).toBe("ToolCallLimitExceeded")
279  })
280})
281 
282describe("Promise.allSettled", () => {
283  test("reports fulfilled and rejected outcomes with catch-normalized reasons", async () => {
284    expect(
285      await value(`
286      return await Promise.allSettled([
287        tools.host.sleepy({ id: 5 }),
288        tools.host.fail({}),
289        "plain",
290        Promise.reject(new Error("boom")),
291      ])
292    `),
293    ).toEqual([
294      { status: "fulfilled", value: 5 },
295      { status: "rejected", reason: { name: "Error", message: "Lookup refused" } },
296      { status: "fulfilled", value: "plain" },
297      { status: "rejected", reason: { name: "Error", message: "boom" } },
298    ])
299  })
300 
301  test("never rejects for program-level failures", async () => {
302    const result = await run(`
303      const settled = await Promise.allSettled([tools.host.fail({}), tools.host.fail({})])
304      return settled.filter((s) => s.status === "rejected").length
305    `)
306    expect(result.ok).toBe(true)
307    if (result.ok) expect(result.value).toBe(2)
308  })
309})
310 
311describe("Promise.race", () => {
312  test("first settlement wins and losers are interrupted", async () => {
313    const trace = makeTrace()
314    const result = await value(
315      `
316        const fast = tools.host.sleepy({ id: 1, ms: 10 })
317        const slow = tools.host.sleepy({ id: 2, ms: 5000 })
318        return await Promise.race([fast, slow])
319      `,
320      { trace },
321    )
322    expect(result).toBe(1)
323    expect(trace.interrupted).toBe(1)
324    expect(trace.completed).toBe(1)
325  })
326 
327  test("awaiting an interrupted loser afterwards is a catchable program failure", async () => {
328    expect(
329      await value(`
330      const fast = tools.host.sleepy({ id: 1, ms: 10 })
331      const slow = tools.host.sleepy({ id: 2, ms: 5000 })
332      const winner = await Promise.race([fast, slow])
333      try {
334        await slow
335        return "no"
336      } catch (e) {
337        return { winner, caught: e.message }
338      }
339    `),
340    ).toEqual({
341      winner: 1,
342      caught: "This tool call was interrupted because another value settled a Promise.race first.",
343    })
344  })
345 
346  test("a rejection can win the race", async () => {
347    expect(
348      await value(`
349      try {
350        await Promise.race([tools.host.fail({}), tools.host.sleepy({ id: 1, ms: 5000 })])
351        return "no"
352      } catch (e) {
353        return e.message
354      }
355    `),
356    ).toBe("Lookup refused")
357  })
358 
359  test("a plain value wins over pending promises", async () => {
360    const trace = makeTrace()
361    expect(
362      await value(`return await Promise.race([tools.host.sleepy({ id: 1, ms: 5000 }), "immediate"])`, { trace }),
363    ).toBe("immediate")
364    expect(trace.interrupted).toBe(1)
365  })
366 
367  test("an empty race is a clear error instead of hanging", async () => {
368    const diagnostic = await error(`return await Promise.race([])`)
369    expect(diagnostic.message).toContain("never settle")
370  })
371})
372 
373describe("Promise.resolve / Promise.reject", () => {
374  test("resolve wraps plain values and passes promises through", async () => {
375    expect(await value(`return await Promise.resolve(42)`)).toBe(42)
376    expect(await value(`return await Promise.resolve(Promise.resolve("nested"))`)).toBe("nested")
377    expect(await value(`return await Promise.resolve(tools.host.sleepy({ id: 3 }))`)).toBe(3)
378  })
379 
380  test("reject produces a promise whose await throws the reason", async () => {
381    expect(
382      await value(`
383      try {
384        await Promise.reject("nope")
385        return "no"
386      } catch (e) {
387        return e
388      }
389    `),
390    ).toBe("nope")
391  })
392})
393 
394describe("timeout interruption of forked calls", () => {
395  test("the execution timeout interrupts in-flight forked fibers", async () => {
396    const trace = makeTrace()
397    const result = await run(
398      `
399        const a = tools.host.sleepy({ id: 1, ms: 60000 })
400        const b = tools.host.sleepy({ id: 2, ms: 60000 })
401        return await a
402      `,
403      { trace, limits: { timeoutMs: 100 } },
404    )
405    expect(result.ok).toBe(false)
406    if (result.ok) return
407    expect(result.error.kind).toBe("TimeoutExceeded")
408    // Both calls started; neither escaped the timeout - the awaited one AND the abandoned one.
409    expect(trace.starts).toEqual([1, 2])
410    expect(trace.interrupted).toBe(2)
411    expect(trace.completed).toBe(0)
412  })
413 
414  test("the timeout also interrupts calls inside Promise.all", async () => {
415    const trace = makeTrace()
416    const result = await run(
417      `return await Promise.all([tools.host.sleepy({ id: 1, ms: 60000 }), tools.host.sleepy({ id: 2, ms: 60000 })])`,
418      { trace, limits: { timeoutMs: 100 } },
419    )
420    expect(result.ok).toBe(false)
421    if (result.ok) return
422    expect(result.error.kind).toBe("TimeoutExceeded")
423    expect(trace.interrupted).toBe(2)
424  })
425})
426 
427describe("unsupported promise surface", () => {
428  test(".then/.catch/.finally give a clear await-instead error", async () => {
429    for (const method of ["then", "catch", "finally"]) {
430      const diagnostic = await error(`return tools.host.sleepy({ id: 1 }).${method}((x) => x)`)
431      expect(diagnostic.kind).toBe("UnsupportedSyntax")
432      expect(diagnostic.message).toContain(`Promise.prototype.${method} is not supported`)
433      expect(diagnostic.message).toContain("await")
434    }
435  })
436 
437  test("other property reads on a promise hint at the missing await", async () => {
438    const diagnostic = await error(`return tools.host.sleepy({ id: 1 }).value`)
439    expect(diagnostic.kind).toBe("InvalidDataValue")
440    expect(diagnostic.message).toContain("un-awaited Promise")
441    expect(diagnostic.message).toContain("await it first")
442  })
443 
444  test("unknown Promise statics list what is available", async () => {
445    const diagnostic = await error(`return await Promise.any([tools.host.sleepy({ id: 1 })])`)
446    expect(diagnostic.message).toContain("Promise.any is not available")
447    expect(diagnostic.message).toContain("Promise.allSettled")
448  })
449 
450  test("new Promise(...) points at tool calls instead", async () => {
451    const diagnostic = await error(`return new Promise((resolve) => resolve(1))`)
452    expect(diagnostic.kind).toBe("UnsupportedSyntax")
453    expect(diagnostic.message).toContain("new Promise(...) is not supported")
454    expect(diagnostic.message).toContain("already return promises")
455  })
456})
457