Skip to content

agent input

ThreadItemConverter

Converts thread items to Agent SDK input items. Widgets, Tasks, and Workflows have default conversions but can be customized. Attachments, Tags, and HiddenContextItems require custom handling based on the use case. Other item types are converted automatically.

Source code in chatkit/agent_input.py
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
class ThreadItemConverter:
    """
    Converts thread items to Agent SDK input items.
    Widgets, Tasks, and Workflows have default conversions but can be customized.
    Attachments, Tags, and HiddenContextItems require custom handling based on the use case.
    Other item types are converted automatically.
    """

    async def attachment_to_message_content(
        self, attachment: Attachment
    ) -> ResponseInputContentParam:
        """
        Convert an attachment in a user message into a message content part to send to the model.
        Required when attachments are enabled.
        """
        raise NotImplementedError(
            "An Attachment was included in a UserMessageItem but Converter.attachment_to_message_content was not implemented"
        )

    async def tag_to_message_content(
        self, tag: UserMessageTagContent
    ) -> ResponseInputContentParam:
        """
        Convert a tag in a user message into a message content part to send to the model as context.
        Required when tags are used.
        """
        raise NotImplementedError(
            "A Tag was included in a UserMessageItem but Converter.tag_to_message_content is not implemented"
        )

    async def generated_image_to_input(
        self, item: GeneratedImageItem
    ) -> TResponseInputItem | list[TResponseInputItem] | None:
        """
        Convert a GeneratedImageItem into input item(s) to send to the model.
        Override this method to customize the conversion of generated images, such as when your
        generated image url is not publicly reachable.
        """
        if not item.image:
            return None

        return Message(
            type="message",
            content=[
                ResponseInputTextParam(
                    type="input_text",
                    text="The following image was generated by the agent.",
                ),
                ResponseInputImageParam(
                    type="input_image",
                    detail="auto",
                    image_url=item.image.url,
                ),
            ],
            role="user",
        )

    async def hidden_context_to_input(
        self, item: HiddenContextItem
    ) -> TResponseInputItem | list[TResponseInputItem] | None:
        """
        Convert a HiddenContextItem into input item(s) to send to the model.
        Required to override when HiddenContextItems with non-string content are used.
        """
        if not isinstance(item.content, str):
            raise NotImplementedError(
                "HiddenContextItems with non-string content were present in a user message but a Converter.hidden_context_to_input was not implemented"
            )

        return _hidden_context_message(item.content)

    async def sdk_hidden_context_to_input(
        self, item: SDKHiddenContextItem
    ) -> TResponseInputItem | list[TResponseInputItem] | None:
        """
        Convert a SDKHiddenContextItem into input item to send to the model.
        This is used by the ChatKit Python SDK for storing additional context
        for internal operations.
        Override if you want to wrap the content in a different format.
        """
        return _hidden_context_message(item.content)

    async def task_to_input(
        self, item: TaskItem
    ) -> TResponseInputItem | list[TResponseInputItem] | None:
        """
        Convert a TaskItem into input item(s) to send to the model.
        """
        return _task_message(item.task)

    async def structured_input_to_input(
        self, item: StructuredInputItem
    ) -> TResponseInputItem | list[TResponseInputItem] | None:
        """
        Convert a StructuredInputItem into input item(s) to send to the model.
        """
        lines = []
        for question in item.inputs:
            answer = question.answer
            if answer is None:
                lines.append(f"- {question.question}: unanswered")
            elif answer.skipped:
                lines.append(f"- {question.question}: skipped")
            else:
                lines.append(f"- {question.question}: {', '.join(answer.values)}")

        text = (
            "A structured input request was displayed to the user with the following "
            f"status: {item.status}\n<StructuredInput>\n"
            + "\n".join(lines)
            + "\n</StructuredInput>"
        )
        return _text_message(text)

    async def workflow_to_input(
        self, item: WorkflowItem
    ) -> TResponseInputItem | list[TResponseInputItem] | None:
        """
        Describe visible custom workflow tasks as model input.
        """
        messages: list[TResponseInputItem] = []
        for task in item.workflow.tasks:
            message = _task_message(task)
            if message is not None:
                messages.append(message)
        return messages

    async def widget_to_input(
        self, item: WidgetItem
    ) -> TResponseInputItem | list[TResponseInputItem] | None:
        """
        Convert a WidgetItem into input item(s) to send to the model.
        By default, WidgetItems converted to a text description with a JSON representation of the widget.
        """
        return _text_message(
            f"The following graphical UI widget (id: {item.id}) was displayed to the user:"
            + item.widget.model_dump_json(exclude_unset=True, exclude_none=True)
        )

    async def user_message_to_input(
        self, item: UserMessageItem, is_last_message: bool = True
    ) -> TResponseInputItem | list[TResponseInputItem] | None:
        # Build the user text exactly as typed, rendering tags as @key
        message_text_parts: list[str] = []
        # Resolve each displayed tag once, in the order the user mentioned it.
        tags: dict[str, UserMessageTagContent] = {}

        for part in item.content:
            if isinstance(part, UserMessageTextContent):
                message_text_parts.append(part.text)
            elif isinstance(part, UserMessageTagContent):
                message_text_parts.append(f"@{part.text}")
                tags.setdefault(part.text, part)
            else:
                assert_never(part)

        user_text_item = Message(
            role="user",
            type="message",
            content=[
                ResponseInputTextParam(
                    type="input_text", text="".join(message_text_parts)
                ),
                *[
                    await self.attachment_to_message_content(a)
                    for a in item.attachments
                ],
            ],
        )

        context_items: list[TResponseInputItem] = []
        if item.quoted_text and is_last_message:
            context_items.append(
                _text_message(
                    f"The user is referring to this in particular: \n{item.quoted_text}"
                )
            )

        if tags:
            tag_content: ResponseInputMessageContentListParam = [
                await self.tag_to_message_content(tag) for tag in tags.values()
            ]
            context_items.append(
                Message(
                    role="user",
                    type="message",
                    content=[
                        ResponseInputTextParam(
                            type="input_text",
                            text=cleandoc("""
                                # User-provided context for @-mentions
                                - When referencing resolved entities, use their canonical names **without** '@'.
                                - The '@' form appears only in user text and should not be echoed.
                            """).strip(),
                        ),
                        *tag_content,
                    ],
                )
            )

        return [user_text_item, *context_items]

    async def assistant_message_to_input(
        self, item: AssistantMessageItem
    ) -> TResponseInputItem | list[TResponseInputItem] | None:
        message = EasyInputMessageParam(
            type="message",
            content="".join(part.text for part in item.content),
            role="assistant",
        )
        if item.phase is not None:
            message["phase"] = item.phase
        return message

    async def client_tool_call_to_input(
        self, item: ClientToolCallItem
    ) -> TResponseInputItem | list[TResponseInputItem] | None:
        """Skip browser tool display records. Use client_tool_continuation_input to resume."""
        return None

    async def end_of_turn_to_input(
        self, item: EndOfTurnItem
    ) -> TResponseInputItem | list[TResponseInputItem] | None:
        # Only used for UI hints - you shouldn't need to override this
        return None

    async def _thread_item_to_input_item(
        self,
        item: ThreadItem,
        is_last_message: bool = True,
    ) -> list[TResponseInputItem]:
        match item:
            case UserMessageItem():
                converted = await self.user_message_to_input(item, is_last_message)
            case AssistantMessageItem():
                converted = await self.assistant_message_to_input(item)
            case ClientToolCallItem():
                converted = await self.client_tool_call_to_input(item)
            case EndOfTurnItem():
                converted = await self.end_of_turn_to_input(item)
            case WidgetItem():
                converted = await self.widget_to_input(item)
            case WorkflowItem():
                converted = await self.workflow_to_input(item)
            case TaskItem():
                converted = await self.task_to_input(item)
            case StructuredInputItem():
                converted = await self.structured_input_to_input(item)
            case HiddenContextItem():
                converted = await self.hidden_context_to_input(item)
            case SDKHiddenContextItem():
                converted = await self.sdk_hidden_context_to_input(item)
            case GeneratedImageItem():
                converted = await self.generated_image_to_input(item)
            case _:
                assert_never(item)

        if converted is None:
            return []
        return converted if isinstance(converted, list) else [converted]

    async def to_agent_input(
        self,
        thread_items: Sequence[ThreadItem] | ThreadItem,
    ) -> list[TResponseInputItem]:
        # Snapshot the sequence before awaiting application converters.
        items = (
            tuple(thread_items)
            if isinstance(thread_items, Sequence)
            else (thread_items,)
        )
        last_index = len(items) - 1
        output: list[TResponseInputItem] = []
        for index, item in enumerate(items):
            output.extend(
                await self._thread_item_to_input_item(
                    item, is_last_message=index == last_index
                )
            )
        return output

attachment_to_message_content async

attachment_to_message_content(attachment: Attachment) -> ResponseInputContentParam

Convert an attachment in a user message into a message content part to send to the model. Required when attachments are enabled.

Source code in chatkit/agent_input.py
105
106
107
108
109
110
111
112
113
114
async def attachment_to_message_content(
    self, attachment: Attachment
) -> ResponseInputContentParam:
    """
    Convert an attachment in a user message into a message content part to send to the model.
    Required when attachments are enabled.
    """
    raise NotImplementedError(
        "An Attachment was included in a UserMessageItem but Converter.attachment_to_message_content was not implemented"
    )

tag_to_message_content async

tag_to_message_content(tag: UserMessageTagContent) -> ResponseInputContentParam

Convert a tag in a user message into a message content part to send to the model as context. Required when tags are used.

Source code in chatkit/agent_input.py
116
117
118
119
120
121
122
123
124
125
async def tag_to_message_content(
    self, tag: UserMessageTagContent
) -> ResponseInputContentParam:
    """
    Convert a tag in a user message into a message content part to send to the model as context.
    Required when tags are used.
    """
    raise NotImplementedError(
        "A Tag was included in a UserMessageItem but Converter.tag_to_message_content is not implemented"
    )

generated_image_to_input async

generated_image_to_input(item: GeneratedImageItem) -> TResponseInputItem | list[TResponseInputItem] | None

Convert a GeneratedImageItem into input item(s) to send to the model. Override this method to customize the conversion of generated images, such as when your generated image url is not publicly reachable.

Source code in chatkit/agent_input.py
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
async def generated_image_to_input(
    self, item: GeneratedImageItem
) -> TResponseInputItem | list[TResponseInputItem] | None:
    """
    Convert a GeneratedImageItem into input item(s) to send to the model.
    Override this method to customize the conversion of generated images, such as when your
    generated image url is not publicly reachable.
    """
    if not item.image:
        return None

    return Message(
        type="message",
        content=[
            ResponseInputTextParam(
                type="input_text",
                text="The following image was generated by the agent.",
            ),
            ResponseInputImageParam(
                type="input_image",
                detail="auto",
                image_url=item.image.url,
            ),
        ],
        role="user",
    )

hidden_context_to_input async

hidden_context_to_input(item: HiddenContextItem) -> TResponseInputItem | list[TResponseInputItem] | None

Convert a HiddenContextItem into input item(s) to send to the model. Required to override when HiddenContextItems with non-string content are used.

Source code in chatkit/agent_input.py
154
155
156
157
158
159
160
161
162
163
164
165
166
async def hidden_context_to_input(
    self, item: HiddenContextItem
) -> TResponseInputItem | list[TResponseInputItem] | None:
    """
    Convert a HiddenContextItem into input item(s) to send to the model.
    Required to override when HiddenContextItems with non-string content are used.
    """
    if not isinstance(item.content, str):
        raise NotImplementedError(
            "HiddenContextItems with non-string content were present in a user message but a Converter.hidden_context_to_input was not implemented"
        )

    return _hidden_context_message(item.content)

sdk_hidden_context_to_input async

sdk_hidden_context_to_input(item: SDKHiddenContextItem) -> TResponseInputItem | list[TResponseInputItem] | None

Convert a SDKHiddenContextItem into input item to send to the model. This is used by the ChatKit Python SDK for storing additional context for internal operations. Override if you want to wrap the content in a different format.

Source code in chatkit/agent_input.py
168
169
170
171
172
173
174
175
176
177
async def sdk_hidden_context_to_input(
    self, item: SDKHiddenContextItem
) -> TResponseInputItem | list[TResponseInputItem] | None:
    """
    Convert a SDKHiddenContextItem into input item to send to the model.
    This is used by the ChatKit Python SDK for storing additional context
    for internal operations.
    Override if you want to wrap the content in a different format.
    """
    return _hidden_context_message(item.content)

task_to_input async

task_to_input(item: TaskItem) -> TResponseInputItem | list[TResponseInputItem] | None

Convert a TaskItem into input item(s) to send to the model.

Source code in chatkit/agent_input.py
179
180
181
182
183
184
185
async def task_to_input(
    self, item: TaskItem
) -> TResponseInputItem | list[TResponseInputItem] | None:
    """
    Convert a TaskItem into input item(s) to send to the model.
    """
    return _task_message(item.task)

structured_input_to_input async

structured_input_to_input(item: StructuredInputItem) -> TResponseInputItem | list[TResponseInputItem] | None

Convert a StructuredInputItem into input item(s) to send to the model.

Source code in chatkit/agent_input.py
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
async def structured_input_to_input(
    self, item: StructuredInputItem
) -> TResponseInputItem | list[TResponseInputItem] | None:
    """
    Convert a StructuredInputItem into input item(s) to send to the model.
    """
    lines = []
    for question in item.inputs:
        answer = question.answer
        if answer is None:
            lines.append(f"- {question.question}: unanswered")
        elif answer.skipped:
            lines.append(f"- {question.question}: skipped")
        else:
            lines.append(f"- {question.question}: {', '.join(answer.values)}")

    text = (
        "A structured input request was displayed to the user with the following "
        f"status: {item.status}\n<StructuredInput>\n"
        + "\n".join(lines)
        + "\n</StructuredInput>"
    )
    return _text_message(text)

workflow_to_input async

workflow_to_input(item: WorkflowItem) -> TResponseInputItem | list[TResponseInputItem] | None

Describe visible custom workflow tasks as model input.

Source code in chatkit/agent_input.py
211
212
213
214
215
216
217
218
219
220
221
222
async def workflow_to_input(
    self, item: WorkflowItem
) -> TResponseInputItem | list[TResponseInputItem] | None:
    """
    Describe visible custom workflow tasks as model input.
    """
    messages: list[TResponseInputItem] = []
    for task in item.workflow.tasks:
        message = _task_message(task)
        if message is not None:
            messages.append(message)
    return messages

widget_to_input async

widget_to_input(item: WidgetItem) -> TResponseInputItem | list[TResponseInputItem] | None

Convert a WidgetItem into input item(s) to send to the model. By default, WidgetItems converted to a text description with a JSON representation of the widget.

Source code in chatkit/agent_input.py
224
225
226
227
228
229
230
231
232
233
234
async def widget_to_input(
    self, item: WidgetItem
) -> TResponseInputItem | list[TResponseInputItem] | None:
    """
    Convert a WidgetItem into input item(s) to send to the model.
    By default, WidgetItems converted to a text description with a JSON representation of the widget.
    """
    return _text_message(
        f"The following graphical UI widget (id: {item.id}) was displayed to the user:"
        + item.widget.model_dump_json(exclude_unset=True, exclude_none=True)
    )

client_tool_call_to_input async

client_tool_call_to_input(item: ClientToolCallItem) -> TResponseInputItem | list[TResponseInputItem] | None

Skip browser tool display records. Use client_tool_continuation_input to resume.

Source code in chatkit/agent_input.py
311
312
313
314
315
async def client_tool_call_to_input(
    self, item: ClientToolCallItem
) -> TResponseInputItem | list[TResponseInputItem] | None:
    """Skip browser tool display records. Use client_tool_continuation_input to resume."""
    return None

client_tool_continuation_input

client_tool_continuation_input(turn: ClientToolTurn) -> list[TResponseInputItem]

Prepare model input from a completed browser tool's native continuation.

A local continuation supplies the whole native input list. A Responses continuation supplies only the missing outputs and requires the matching provider context.

Source code in chatkit/agent_input.py
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
def client_tool_continuation_input(
    turn: ClientToolTurn,
) -> list[TResponseInputItem]:
    """Prepare model input from a completed browser tool's native continuation.

    A local continuation supplies the whole native input list. A Responses continuation
    supplies only the missing outputs and requires the matching provider context.
    """
    item = turn.tool_call
    invocation = turn.invocation
    if item.status != "completed" or invocation.continuation_turn_id is None:
        raise ValueError("Complete the client tool before preparing its continuation")
    if (item.id, item.thread_id) != (invocation.item_id, invocation.thread_id):
        raise ValueError("The client tool result and private invocation must match")
    if invocation.model_continuation is None:
        raise ValueError("This client tool has no captured native model continuation")

    items = deepcopy(invocation.model_continuation.items)
    matching_outputs = [
        value
        for value in items
        if value.get("type") == "function_call_output"
        and value.get("call_id") == invocation.call_id
    ]
    if len(matching_outputs) != 1:
        raise ValueError(
            "The model continuation must contain exactly one matching output"
        )
    matching_outputs[0]["output"] = json.dumps(item.output, allow_nan=False)
    return cast(list[TResponseInputItem], items)

simple_to_agent_input async

simple_to_agent_input(thread_items: Sequence[ThreadItem] | ThreadItem) -> list[TResponseInputItem]

Helper that converts thread items using the default ThreadItemConverter.

Source code in chatkit/agent_input.py
382
383
384
385
386
async def simple_to_agent_input(
    thread_items: Sequence[ThreadItem] | ThreadItem,
) -> list[TResponseInputItem]:
    """Helper that converts thread items using the default ThreadItemConverter."""
    return await _DEFAULT_CONVERTER.to_agent_input(thread_items)