# Middleware with History Modification Ref: https://docs.langchain.com/oss/python/langchain/middleware/built-in This notebook demonstrates `SummarizationMiddleware`, which automatically summarizes conversation history when approaching token limits. ## Setup Configure `.env` before running. See `.env.sample`. ```python import rich from dotenv import load_dotenv load_dotenv() ``` True ## SummarizationMiddleware Built-in middleware that manages conversation history intelligently: **How it works:** 1. Monitors message count/tokens in `before_model` hook 2. When `trigger` threshold is exceeded, summarizes older messages using an LLM 3. Replaces old messages with a summary, keeping recent messages intact 4. Uses `RemoveMessage(id=REMOVE_ALL_MESSAGES)` to clear history, then adds summary + kept messages **Why this matters - ToolMessage constraint:** LLM APIs require that every `ToolMessage` must have a corresponding `AIMessage` with matching `tool_call_id`. If you naively trim messages and accidentally remove the AIMessage while keeping its ToolMessage (or vice versa), the API call will fail. `SummarizationMiddleware` handles this by scanning for `tool_calls` in AIMessage and their corresponding ToolMessage, adjusting the cutoff point to keep pairs together. **Parameters:** - `model`: LLM used to generate summaries - `trigger`: When to summarize - `("messages", N)`, `("tokens", N)`, or `("fraction", 0.8)` - `keep`: How much to preserve - same format as trigger ```python from datetime import datetime from typing import Any from langchain.agents import create_agent from langchain.agents.middleware import SummarizationMiddleware from langchain_anthropic import ChatAnthropic from langchain_core.runnables import RunnableConfig from langchain_core.tools import tool from langgraph.checkpoint.memory import InMemorySaver from langgraph.graph.state import CompiledStateGraph @tool def get_current_time() -> str: """Get the current time.""" return datetime.now().isoformat() model = ChatAnthropic(model="claude-sonnet-4-5-20250929") checkpointer = InMemorySaver() # Summarize when messages exceed 8, keep last 4 messages summarization = SummarizationMiddleware( model=model, trigger=("messages", 8), keep=("messages", 4), ) agent: CompiledStateGraph[Any] = create_agent( model=model, tools=[get_current_time], system_prompt="You are a helpful assistant. Keep your responses brief.", checkpointer=checkpointer, middleware=[summarization], ) ``` ## Test Summarization Send multiple messages to trigger the summarization behavior. ```python config: RunnableConfig = {"configurable": {"thread_id": "summarization-test"}} questions = [ "My name is Alice and I'm a software engineer.", "What is 2 + 2?", "What is the capital of France?", "What time is it?", "Tell me a fun fact about cats.", "What is my name and profession?", # Should be preserved in summary ] for q in questions: rich.print(f"[bold cyan]User: {q}[/bold cyan]") response = agent.invoke({"messages": [{"role": "user", "content": q}]}, config) ai_message = response["messages"][-1] rich.print(f"[bold green]Assistant: {ai_message.content}[/bold green]") rich.print(f"[dim]Total messages in state: {len(response['messages'])}[/dim]") ``` User: My name is Alice and I'm a software engineer. Assistant: Nice to meet you, Alice! It's great to connect with a fellow software engineer. Is there anything I can help you with today? Total messages in state: 2 User: What is 2 + 2? Assistant: 2 + 2 = 4 Total messages in state: 4 User: What is the capital of France? Assistant: The capital of France is Paris. Total messages in state: 6 User: What time is it? Assistant: The current time is **7:37 AM** (07:37) on December 5th, 2025. Total messages in state: 6 User: Tell me a fun fact about cats. Assistant: Here's a fun fact about cats: Cats spend about 70% of their lives sleeping, which means the average cat sleeps for around 13-16 hours per day. They're basically professional nappers! 😺 Total messages in state: 8 User: What is my name and profession? Assistant: Based on the conversation summary, your name is **Alice** and you're a **software engineer**! 👩‍💻 Total messages in state: 6 ## Inspect the Summary Check what the summarized history looks like. ```python state = agent.get_state(config) messages = state.values.get("messages", []) rich.print(f"[bold]Current message count: {len(messages)}[/bold]") rich.print("messages =", messages) ``` Current message count: 6 messages = [ HumanMessage( content='Here is a summary of the conversation to date:\n\nUser name: Alice\nProfession: Software engineer\nPrevious questions answered: 2 + 2 = 4\nCapital of France: Paris\nCurrent time retrieved: 2025-12-05T07:37:37.658663', additional_kwargs={}, response_metadata={}, id='5104b7b4-d90f-4085-9eb1-dbef3c2771ba' ), AIMessage( content='The current time is **7:37 AM** (07:37) on December 5th, 2025.', additional_kwargs={}, response_metadata={ 'id': 'msg_01LaSQpqFtJ9St9HGDx6c7xU', 'model': 'claude-sonnet-4-5-20250929', 'stop_reason': 'end_turn', 'stop_sequence': None, 'usage': { 'cache_creation': {'ephemeral_1h_input_tokens': 0, 'ephemeral_5m_input_tokens': 0}, 'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'input_tokens': 675, 'output_tokens': 29, 'server_tool_use': None, 'service_tier': 'standard' }, 'model_name': 'claude-sonnet-4-5-20250929', 'model_provider': 'anthropic' }, id='lc_run--0856857e-c768-4e3d-af6b-497dec276377-0', usage_metadata={ 'input_tokens': 675, 'output_tokens': 29, 'total_tokens': 704, 'input_token_details': { 'cache_creation': 0, 'cache_read': 0, 'ephemeral_5m_input_tokens': 0, 'ephemeral_1h_input_tokens': 0 } } ), HumanMessage( content='Tell me a fun fact about cats.', additional_kwargs={}, response_metadata={}, id='e1385054-82ee-462b-beb1-151b08a89ccc' ), AIMessage( content="Here's a fun fact about cats: Cats spend about 70% of their lives sleeping, which means the average cat sleeps for around 13-16 hours per day. They're basically professional nappers! 😺", additional_kwargs={}, response_metadata={ 'id': 'msg_01W6AkptcYJKw16t89Gp37u9', 'model': 'claude-sonnet-4-5-20250929', 'stop_reason': 'end_turn', 'stop_sequence': None, 'usage': { 'cache_creation': {'ephemeral_1h_input_tokens': 0, 'ephemeral_5m_input_tokens': 0}, 'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'input_tokens': 715, 'output_tokens': 51, 'server_tool_use': None, 'service_tier': 'standard' }, 'model_name': 'claude-sonnet-4-5-20250929', 'model_provider': 'anthropic' }, id='lc_run--ffa3c8a2-05f1-4ff4-bc9c-efc0c1bdf406-0', usage_metadata={ 'input_tokens': 715, 'output_tokens': 51, 'total_tokens': 766, 'input_token_details': { 'cache_creation': 0, 'cache_read': 0, 'ephemeral_5m_input_tokens': 0, 'ephemeral_1h_input_tokens': 0 } } ), HumanMessage( content='What is my name and profession?', additional_kwargs={}, response_metadata={}, id='d206354e-69e9-4f91-8423-c721a16d1bce' ), AIMessage( content="Based on the conversation summary, your name is **Alice** and you're a **software engineer**! 👩\u200d💻", additional_kwargs={}, response_metadata={ 'id': 'msg_0146qHgo7RpQ5stvkedRuBbk', 'model': 'claude-sonnet-4-5-20250929', 'stop_reason': 'end_turn', 'stop_sequence': None, 'usage': { 'cache_creation': {'ephemeral_1h_input_tokens': 0, 'ephemeral_5m_input_tokens': 0}, 'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'input_tokens': 719, 'output_tokens': 31, 'server_tool_use': None, 'service_tier': 'standard' }, 'model_name': 'claude-sonnet-4-5-20250929', 'model_provider': 'anthropic' }, id='lc_run--088372cf-17e6-4e47-8f1e-335b25a5bad3-0', usage_metadata={ 'input_tokens': 719, 'output_tokens': 31, 'total_tokens': 750, 'input_token_details': { 'cache_creation': 0, 'cache_read': 0, 'ephemeral_5m_input_tokens': 0, 'ephemeral_1h_input_tokens': 0 } } ) ] ## Custom History Modification You can also write custom middleware to modify history. The key is to return a dict with `messages` from `before_model` hook. **Important:** When modifying messages, ensure AIMessage/ToolMessage pairs stay together. ```python from langchain.agents.middleware.types import AgentMiddleware, AgentState from langchain_core.messages import HumanMessage from langgraph.runtime import Runtime class SystemPromptInjectionMiddleware(AgentMiddleware[AgentState[Any], None]): """Middleware that injects dynamic context into conversation history.""" def __init__(self, context_provider: str) -> None: self.context_provider = context_provider def before_model(self, state: AgentState[Any], runtime: Runtime[None]) -> dict[str, Any] | None: messages = state["messages"] # Only inject right after user message (not after tool calls) if not messages or not isinstance(messages[-1], HumanMessage): return None context_message = HumanMessage( content=f"[System context: {self.context_provider}]", ) rich.print(f"[bold magenta]Injected context: {self.context_provider}[/bold magenta]") return {"messages": [context_message, *messages]} # Example: inject context that user always wants to know the time checkpointer3 = InMemorySaver() agent3: CompiledStateGraph[Any] = create_agent( model=model, tools=[get_current_time], system_prompt="You are a helpful assistant.", checkpointer=checkpointer3, middleware=[SystemPromptInjectionMiddleware("User always wants to know the current time")], ) config3: RunnableConfig = {"configurable": {"thread_id": "custom-test"}} response = agent3.invoke({"messages": [{"role": "user", "content": "Hello!"}]}, config3) rich.print("response =", response) ``` Injected context: User always wants to know the current time response = { 'messages': [ HumanMessage( content='Hello!', additional_kwargs={}, response_metadata={}, id='5df86bbc-4fdd-4c58-8c1d-155299cfa89e' ), HumanMessage( content='[System context: User always wants to know the current time]', additional_kwargs={}, response_metadata={}, id='8f8bdf67-9658-4475-9d91-848033d31183' ), AIMessage( content=[ { 'id': 'toolu_01XSaQkb1tjFSJu9phHt4yo7', 'input': {}, 'name': 'get_current_time', 'type': 'tool_use' } ], additional_kwargs={}, response_metadata={ 'id': 'msg_01P4Mxcpe3kBw6gD4UtzTawG', 'model': 'claude-sonnet-4-5-20250929', 'stop_reason': 'tool_use', 'stop_sequence': None, 'usage': { 'cache_creation': {'ephemeral_1h_input_tokens': 0, 'ephemeral_5m_input_tokens': 0}, 'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'input_tokens': 564, 'output_tokens': 38, 'server_tool_use': None, 'service_tier': 'standard' }, 'model_name': 'claude-sonnet-4-5-20250929', 'model_provider': 'anthropic' }, id='lc_run--b0643102-1d4c-4f82-ab84-80e69b0c12e8-0', tool_calls=[ { 'name': 'get_current_time', 'args': {}, 'id': 'toolu_01XSaQkb1tjFSJu9phHt4yo7', 'type': 'tool_call' } ], usage_metadata={ 'input_tokens': 564, 'output_tokens': 38, 'total_tokens': 602, 'input_token_details': { 'cache_read': 0, 'cache_creation': 0, 'ephemeral_5m_input_tokens': 0, 'ephemeral_1h_input_tokens': 0 } } ), ToolMessage( content='2025-12-05T07:37:55.547538', name='get_current_time', id='9b03c5b5-2812-4838-8aa0-5803a03f4c2f', tool_call_id='toolu_01XSaQkb1tjFSJu9phHt4yo7' ), AIMessage( content='Hello! 👋 The current time is **7:37 AM (UTC)** on December 5th, 2025. How can I help you today?', additional_kwargs={}, response_metadata={ 'id': 'msg_018RCq6EZZHom9LHkNGeByUL', 'model': 'claude-sonnet-4-5-20250929', 'stop_reason': 'end_turn', 'stop_sequence': None, 'usage': { 'cache_creation': {'ephemeral_1h_input_tokens': 0, 'ephemeral_5m_input_tokens': 0}, 'cache_creation_input_tokens': 0, 'cache_read_input_tokens': 0, 'input_tokens': 630, 'output_tokens': 40, 'server_tool_use': None, 'service_tier': 'standard' }, 'model_name': 'claude-sonnet-4-5-20250929', 'model_provider': 'anthropic' }, id='lc_run--33ca1f81-5253-4b81-a7ed-e9e548f3b402-0', usage_metadata={ 'input_tokens': 630, 'output_tokens': 40, 'total_tokens': 670, 'input_token_details': { 'cache_read': 0, 'cache_creation': 0, 'ephemeral_5m_input_tokens': 0, 'ephemeral_1h_input_tokens': 0 } } ) ] }