Created
September 13, 2024 15:14
-
-
Save kwnath/f42737c023767d5effdcca20cb5bd0a6 to your computer and use it in GitHub Desktop.
anthropic_streaming_client.ipynb
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "# Anthropic Streaming\n", | |
| "\n", | |
| "The following use examples from the [Anthropic sdk readme](https://github.com/anthropics/anthropic-sdk-python?tab=readme-ov-file#tool-use) that link to tool use (function calling) [here](https://docs.anthropic.com/en/docs/build-with-claude/tool-use)" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "# First initialise the client\n", | |
| "\n", | |
| "from anthropic import Anthropic\n", | |
| "import os \n", | |
| "\n", | |
| "client = Anthropic(api_key=os.environ.get(\"ANTHROPIC_API_KEY\"))" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "# Then let's try the expected outcome:\n", | |
| "\n", | |
| "message = client.messages.create(\n", | |
| " model=\"claude-3-5-sonnet-20240620\",\n", | |
| " max_tokens=2048,\n", | |
| " temperature=0.0,\n", | |
| " tools=[\n", | |
| " {\n", | |
| " \"name\": \"get_weather\",\n", | |
| " \"description\": \"Get the current weather in a given location\",\n", | |
| " \"input_schema\": {\n", | |
| " \"type\": \"object\",\n", | |
| " \"properties\": {\n", | |
| " \"location\": {\n", | |
| " \"type\": \"string\",\n", | |
| " \"description\": \"The city and state, e.g. San Francisco, CA\",\n", | |
| " }\n", | |
| " },\n", | |
| " \"required\": [\"location\"],\n", | |
| " },\n", | |
| " }\n", | |
| " ],\n", | |
| " stream=False,\n", | |
| " messages=[{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco?\"}],\n", | |
| ")\n", | |
| "\n", | |
| "print(message) \n", | |
| "# In the below output we get the tool use and the following chunks... \n", | |
| "# ToolUseBlock(id='toolu_01761DvCk3yuAJZgHWe9ocNZ', input={'location': 'San Francisco, CA'}, name='get_weather', type='tool_use')\n", | |
| "\n", | |
| "\n", | |
| "# [2024-09-13 15:54:37,483] #51119 INFO (httpx) - HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", | |
| "# ('id', 'msg_01LSBCVYJZz824RMhZojNxxE')\n", | |
| "# ('content', [TextBlock(text=\"Certainly! I can help you get the current weather information for San Francisco. To do that, I'll use the get_weather function. Let me fetch that information for you right away.\", type='text'), ToolUseBlock(id='toolu_01761DvCk3yuAJZgHWe9ocNZ', input={'location': 'San Francisco, CA'}, name='get_weather', type='tool_use')])\n", | |
| "# ('model', 'claude-3-5-sonnet-20240620')\n", | |
| "# ('role', 'assistant')\n", | |
| "# ('stop_reason', 'tool_use')\n", | |
| "# ('stop_sequence', None)\n", | |
| "# ('type', 'message')\n", | |
| "# ('usage', Usage(input_tokens=384, output_tokens=95))" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "## Now with streaming" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "stream = client.messages.create(\n", | |
| " model=\"claude-3-5-sonnet-20240620\",\n", | |
| " max_tokens=2048,\n", | |
| " temperature=0.0,\n", | |
| " tools=[\n", | |
| " {\n", | |
| " \"name\": \"get_weather\",\n", | |
| " \"description\": \"Get the current weather in a given location\",\n", | |
| " \"input_schema\": {\n", | |
| " \"type\": \"object\",\n", | |
| " \"properties\": {\n", | |
| " \"location\": {\n", | |
| " \"type\": \"string\",\n", | |
| " \"description\": \"The city and state, e.g. San Francisco, CA\",\n", | |
| " }\n", | |
| " },\n", | |
| " \"required\": [\"location\"],\n", | |
| " },\n", | |
| " }\n", | |
| " ],\n", | |
| " tool_choice={\"type\": \"any\"},\n", | |
| " stream=True,\n", | |
| " messages=[{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco?\"}],\n", | |
| ")\n", | |
| "for event in stream:\n", | |
| " print(event)\n", | |
| "\n", | |
| "\n", | |
| " \n", | |
| "# Only two chunks are being sent back where we expected several.\n", | |
| "# \n", | |
| "# 1. RawMessageStartEvent(message=Message(id='msg_015Ve4hQLg4DjJK5WUBFY2Dc', content=[], model='claude-3-5-sonnet-20240620', role='assistant', stop_reason=None, stop_sequence=None, type='message', usage=Usage(input_tokens=366, output_tokens=10)), type='message_start')\n", | |
| "# 2. RawContentBlockStartEvent(content_block=ToolUseBlock(id='toolu_01Jw9TVHwfUm5BQPXWz5zn15', input={}, name='get_weather', type='tool_use'), index=0, type='content_block_start')\n", | |
| "# \n", | |
| "# We can confirm this if we monkey patch the httpx client. Which you can see in the output below:\n", | |
| "\n", | |
| "# Request body: {\"max_tokens\": 2048, \"messages\": [{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco?\"}], \"model\": \"claude-3-5-sonnet-20240620\", \"stream\": true, \"temperature\": 0.0, \"tool_choice\": {\"type\": \"any\"}, \"tools\": [{\"name\": \"get_weather\", \"description\": \"Get the current weather in a given location\", \"input_schema\": {\"type\": \"object\", \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The city and state, e.g. San Francisco, CA\"}}, \"required\": [\"location\"]}}]}\n", | |
| "# [2024-09-13 16:03:52,173] #68695 INFO (httpx) - HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", | |
| "# Response body: \n", | |
| "# event: message_start\n", | |
| "# data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_01ReJV59o2NGBhuXNVBnNn75\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-3-5-sonnet-20240620\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"usage\":{\"input_tokens\":366,\"output_tokens\":10}} }\n", | |
| "\n", | |
| "# event: content_block_start\n", | |
| "# data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01BKidtwXDb5SXp3qkQLqx3g\",\"name\":\"get_weather\",\"input\":{}} }\n", | |
| "\n", | |
| "# event: ping\n", | |
| "# data: {\"type\": \"ping\"}\n", | |
| "\n", | |
| "# event: content_block_delta\n", | |
| "# data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"} }\n", | |
| "\n", | |
| "# event: content_block_delta\n", | |
| "# data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"lo\"} }\n", | |
| "\n", | |
| "# event: content_block_delta\n", | |
| "# data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"cation\\\": \\\"Sa\"} }\n", | |
| "\n", | |
| "# event: content_block_delta\n", | |
| "# data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"n Francisco\"} }\n", | |
| "\n", | |
| "# event: content_block_delta\n", | |
| "# data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\", C\"} }\n", | |
| "\n", | |
| "# event: content_block_delta\n", | |
| "# data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"A\\\"}\"} }\n", | |
| "\n", | |
| "# event: content_block_stop\n", | |
| "# data: {\"type\":\"content_block_stop\",\"index\":0 }\n", | |
| "\n", | |
| "# event: message_delta\n", | |
| "# data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null},\"usage\":{\"output_tokens\":41} }\n", | |
| "\n", | |
| "# event: message_stop\n", | |
| "# data: {\"type\":\"message_stop\" }" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": {}, | |
| "source": [ | |
| "## Now with streaming and changing the tool choice to 'auto'" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "stream = client.messages.create(\n", | |
| " model=\"claude-3-5-sonnet-20240620\",\n", | |
| " max_tokens=2048,\n", | |
| " temperature=0.0,\n", | |
| " tools=[\n", | |
| " {\n", | |
| " \"name\": \"get_weather\",\n", | |
| " \"description\": \"Get the current weather in a given location\",\n", | |
| " \"input_schema\": {\n", | |
| " \"type\": \"object\",\n", | |
| " \"properties\": {\n", | |
| " \"location\": {\n", | |
| " \"type\": \"string\",\n", | |
| " \"description\": \"The city and state, e.g. San Francisco, CA\",\n", | |
| " }\n", | |
| " },\n", | |
| " \"required\": [\"location\"],\n", | |
| " },\n", | |
| " }\n", | |
| " ],\n", | |
| " tool_choice={\"type\": \"auto\"},\n", | |
| " stream=True,\n", | |
| " messages=[{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco?\"}],\n", | |
| ")\n", | |
| "for event in stream:\n", | |
| " print(event)\n", | |
| " \n", | |
| " \n", | |
| "# However if we change the tool choice to \"auto\" we get back all the chunks but still not inputs for the tool use.\n", | |
| "\n", | |
| "\n", | |
| "# RawMessageStartEvent(message=Message(id='msg_0172tqQYZ6BdoV6eA6JMZS17', content=[], model='claude-3-5-sonnet-20240620', role='assistant', stop_reason=None, stop_sequence=None, type='message', usage=Usage(input_tokens=384, output_tokens=1)), type='message_start')\n", | |
| "# RawContentBlockStartEvent(content_block=TextBlock(text='', type='text'), index=0, type='content_block_start')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text='Certainly', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text='!', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=' I', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=' can', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=' help', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=' you get', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=' the current weather information', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=' for San Francisco.', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=' To', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=' ', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=\"do that, I'll use\", type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=' the get_weather function', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text='.', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=' Let', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=' me fetch', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=' that', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=' information', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=' for you right', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text=' away', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockDeltaEvent(delta=TextDelta(text='.', type='text_delta'), index=0, type='content_block_delta')\n", | |
| "# RawContentBlockStopEvent(index=0, type='content_block_stop')\n", | |
| "# RawContentBlockStartEvent(content_block=ToolUseBlock(id='toolu_01K579Ws9JYaxt4jniaGxBcn', input={}, name='get_weather', type='tool_use'), index=1, type='content_block_start')" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": {}, | |
| "outputs": [], | |
| "source": [ | |
| "from anthropic import AsyncAnthropic\n", | |
| "\n", | |
| "a_client = AsyncAnthropic(api_key=os.environ.get(\"ANTHROPIC_API_KEY\"))\n", | |
| "\n", | |
| "async with a_client.messages.stream(\n", | |
| " model=\"claude-3-5-sonnet-20240620\",\n", | |
| " max_tokens=2048,\n", | |
| " temperature=0.0,\n", | |
| " tools=[\n", | |
| " {\n", | |
| " \"name\": \"get_weather\",\n", | |
| " \"description\": \"Get the current weather in a given location\",\n", | |
| " \"input_schema\": {\n", | |
| " \"type\": \"object\",\n", | |
| " \"properties\": {\n", | |
| " \"location\": {\n", | |
| " \"type\": \"string\",\n", | |
| " \"description\": \"The city and state, e.g. San Francisco, CA\",\n", | |
| " }\n", | |
| " },\n", | |
| " \"required\": [\"location\"],\n", | |
| " },\n", | |
| " }\n", | |
| " ],\n", | |
| " tool_choice={\"type\": \"any\"},\n", | |
| " messages=[{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco?\"}]\n", | |
| ") as stream:\n", | |
| " async for text in stream.text_stream:\n", | |
| " print(text, end=\"\", flush=True)\n", | |
| " print()\n", | |
| " \n", | |
| "message = await stream.get_final_message()\n", | |
| "print(message.to_json())\n", | |
| "\n", | |
| "# This is essentially the same thing as stream=True but it's clearer to see the expected output where the\n", | |
| "# Tool call also provides an input arg.\n", | |
| "\n", | |
| "# Sending async request: POST https://api.anthropic.com/v1/messages\n", | |
| "# Request body: {\"max_tokens\": 2048, \"messages\": [{\"role\": \"user\", \"content\": \"What's the weather like in San Francisco?\"}], \"model\": \"claude-3-5-sonnet-20240620\", \"temperature\": 0.0, \"tools\": [{\"name\": \"get_weather\", \"description\": \"Get the current weather in a given location\", \"input_schema\": {\"type\": \"object\", \"properties\": {\"location\": {\"type\": \"string\", \"description\": \"The city and state, e.g. San Francisco, CA\"}}, \"required\": [\"location\"]}}], \"tool_choice\": {\"type\": \"any\"}, \"stream\": true}\n", | |
| "# [2024-09-13 16:03:53,495] #68695 INFO (httpx) - HTTP Request: POST https://api.anthropic.com/v1/messages \"HTTP/1.1 200 OK\"\n", | |
| "\n", | |
| "# {\n", | |
| "# \"id\": \"msg_01Qsd7oBD9EcoAa12iyQSabZ\",\n", | |
| "# \"content\": [\n", | |
| "# {\n", | |
| "# \"id\": \"toolu_01NKLPqDLZFYa5sXvYRfPUwp\",\n", | |
| "# \"input\": {\n", | |
| "# \"location\": \"San Francisco, CA\"\n", | |
| "# },\n", | |
| "# \"name\": \"get_weather\",\n", | |
| "# \"type\": \"tool_use\"\n", | |
| "# }\n", | |
| "# ],\n", | |
| "# \"model\": \"claude-3-5-sonnet-20240620\",\n", | |
| "# \"role\": \"assistant\",\n", | |
| "# \"stop_reason\": \"tool_use\",\n", | |
| "# \"stop_sequence\": null,\n", | |
| "# \"type\": \"message\",\n", | |
| "# \"usage\": {\n", | |
| "# \"input_tokens\": 366,\n", | |
| "# \"output_tokens\": 41\n", | |
| "# }\n", | |
| "# }" | |
| ] | |
| } | |
| ], | |
| "metadata": { | |
| "language_info": { | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "file_extension": ".py", | |
| "mimetype": "text/x-python", | |
| "name": "python", | |
| "nbconvert_exporter": "python", | |
| "pygments_lexer": "ipython3", | |
| "version": "3.11.7" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 2 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment