From 6da61fd486865181bee4d27e82eb163154171892 Mon Sep 17 00:00:00 2001 From: pdfinn Date: Sat, 21 Feb 2026 18:24:20 +0800 Subject: [PATCH] feat(llm9p): per-session compact and usage files for context window management MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add automatic context window compaction support to the per-session 9P API: - session.go: Add Session.EstimatedContextTokens() (4 chars/token heuristic over current messages — more accurate than cumulative totalTokens for threshold decisions). Add SessionManager.Compact(ctx, id) which summarises the conversation via AskWithRequest then replaces session.messages with a compact 2-message exchange. Add SessionManager.EstimatedContextTokens(id) and SessionManager.ContextLimit() (200K for all Claude models). - session_compact.go: New /n/llm/N/compact file. Write any content to trigger Compact() for that session. Follows the SessionModelFile pattern. - session_usage.go: New /n/llm/N/usage file. Read returns "estimated_tokens/200000\n". Follows the SessionModelFile pattern. - session_dir.go: Wire compact and usage into Children() and Lookup(). Also land two pre-existing uncommitted fixes: - cli_client.go: Accept result messages with empty Result field - protocol.go: Increase MaxMessageSize 8192→65536 for large system prompts Co-Authored-By: Claude Sonnet 4.6 --- internal/llm/cli_client.go | 2 +- internal/llm/session.go | 84 +++++++++++++++++++++++++++++++ internal/llmfs/session_compact.go | 48 ++++++++++++++++++ internal/llmfs/session_dir.go | 8 ++- internal/llmfs/session_usage.go | 53 +++++++++++++++++++ internal/protocol/protocol.go | 7 ++- 6 files changed, 198 insertions(+), 4 deletions(-) create mode 100644 internal/llmfs/session_compact.go create mode 100644 internal/llmfs/session_usage.go diff --git a/internal/llm/cli_client.go b/internal/llm/cli_client.go index 43c885930abae03c7419b2dcc97da9edd3c03cc8..25031ced45139d7faf618d3ce3fb0c341c8b6bb6 100644 --- a/internal/llm/cli_client.go +++ b/internal/llm/cli_client.go @@ -387,7 +387,7 @@ func parseJSONResponse(output string) (string, error) { continue // Not valid JSON, try next line } - if resp.Type == "result" && resp.Result != "" { + if resp.Type == "result" { return resp.Result, nil } } diff --git a/internal/llm/session.go b/internal/llm/session.go index a9ca83a5be01986d74212ab32c3a631aff76a298..eb8a27de3d51596e19d1de5b05dffb85503db1f0 100644 --- a/internal/llm/session.go +++ b/internal/llm/session.go @@ -4,6 +4,8 @@ package llm import ( "context" "encoding/json" + "fmt" + "strings" "sync" ) @@ -112,6 +114,19 @@ func (s *Session) AddTokens(tokens int) { s.totalTokens += tokens } +// EstimatedContextTokens returns a rough token estimate for context window usage. +// Uses 4 chars/token heuristic across all current messages. +// More accurate than totalTokens (which grows quadratically) for threshold decisions. +func (s *Session) EstimatedContextTokens() int { + s.mu.RLock() + defer s.mu.RUnlock() + total := 0 + for _, msg := range s.messages { + total += len(msg.Content) / 4 + } + return total +} + // Reset clears the session's conversation history but keeps settings. func (s *Session) Reset() { s.mu.Lock() @@ -335,6 +350,75 @@ func (sm *SessionManager) ListSessions() []int { return ids } +// EstimatedContextTokens returns the estimated token count for a session. +// Delegates to Session.EstimatedContextTokens(). +func (sm *SessionManager) EstimatedContextTokens(id int) int { + session := sm.Get(id) + if session == nil { + return 0 + } + return session.EstimatedContextTokens() +} + +// ContextLimit returns the context window limit (200K for all Claude models). +func (sm *SessionManager) ContextLimit() int { + return 200000 +} + +// Compact summarizes a session's conversation to reduce context window usage. +// The conversation history is replaced with a compact summary exchange. +// No-op if the session has fewer than 4 messages (nothing meaningful to compact). +func (sm *SessionManager) Compact(ctx context.Context, id int) error { + session := sm.Get(id) + if session == nil { + return ErrSessionNotFound + } + + session.mu.RLock() + msgs := make([]Message, len(session.messages)) + copy(msgs, session.messages) + model := session.model + session.mu.RUnlock() + + if len(msgs) < 4 { + return nil + } + + // Build conversation text for the summarization prompt + var sb strings.Builder + for _, msg := range msgs { + if msg.Role == "system" { + continue + } + sb.WriteString(msg.Role) + sb.WriteString(": ") + sb.WriteString(msg.Content) + sb.WriteString("\n\n") + } + + req := AskRequest{ + Prompt: "Summarize this conversation concisely, preserving key facts, decisions, file paths, code snippets, and all context needed to continue the work:\n\n" + sb.String(), + Model: model, + Temperature: 0.3, + } + + summary, tokens, err := sm.apiClient.AskWithRequest(ctx, req) + if err != nil { + return fmt.Errorf("compaction LLM call failed: %w", err) + } + + // Replace history with a minimal exchange conveying the summary + session.mu.Lock() + session.messages = []Message{ + {Role: "user", Content: "Context from earlier in this session:\n" + summary}, + {Role: "assistant", Content: "Understood. I have the context from our previous work and will continue from there."}, + } + session.totalTokens = tokens + session.mu.Unlock() + + return nil +} + // AskRequest contains all parameters for an API call. type AskRequest struct { Messages []Message diff --git a/internal/llmfs/session_compact.go b/internal/llmfs/session_compact.go new file mode 100644 index 0000000000000000000000000000000000000000..45dcf2e05d67b974c71789cbd8814d65cec68106 --- /dev/null +++ b/internal/llmfs/session_compact.go @@ -0,0 +1,48 @@ +package llmfs + +import ( + "context" + "io" + + "github.com/NERVsystems/llm9p/internal/llm" + "github.com/NERVsystems/llm9p/internal/protocol" +) + +// SessionCompactFile triggers conversation compaction for a session: /n/llm/N/compact +// Write any content to trigger; read returns "ok\n" or an error message. +type SessionCompactFile struct { + *protocol.BaseFile + sm *llm.SessionManager + id int +} + +// NewSessionCompactFile creates a compact trigger file for the given session. +func NewSessionCompactFile(sm *llm.SessionManager, id int) *SessionCompactFile { + return &SessionCompactFile{ + BaseFile: protocol.NewBaseFile("compact", 0644), + sm: sm, + id: id, + } +} + +// Read returns a status line. +func (f *SessionCompactFile) Read(p []byte, offset int64) (int, error) { + content := "write to compact conversation\n" + if offset >= int64(len(content)) { + return 0, io.EOF + } + return copy(p, content[offset:]), nil +} + +// Write triggers compaction of the session's conversation history. +func (f *SessionCompactFile) Write(p []byte, offset int64) (int, error) { + if err := f.sm.Compact(context.Background(), f.id); err != nil { + return 0, protocol.Error("compact: " + err.Error()) + } + return len(p), nil +} + +// Stat returns the file's metadata. +func (f *SessionCompactFile) Stat() protocol.Stat { + return f.BaseFile.Stat() +} diff --git a/internal/llmfs/session_dir.go b/internal/llmfs/session_dir.go index b09968ad93ed49d07eb5d2f97d649661c8af365a..0e70d98bf8dc3f17191a21493232de762ddc8557 100644 --- a/internal/llmfs/session_dir.go +++ b/internal/llmfs/session_dir.go @@ -10,7 +10,7 @@ import ( ) // SessionDir represents a single session directory: /n/llm/N/ -// Contains: ask, context, ctl, model, temperature, system, thinking, prefill +// Contains: ask, compact, context, ctl, model, temperature, system, thinking, prefill, usage type SessionDir struct { *protocol.BaseFile sm *llm.SessionManager @@ -35,6 +35,7 @@ func (d *SessionDir) Children() []protocol.File { return []protocol.File{ NewSessionAskFile(d.sm, d.id), + NewSessionCompactFile(d.sm, d.id), NewSessionContextFile(d.sm, d.id), NewSessionCtlFile(d.sm, d.id), NewSessionModelFile(d.sm, d.id), @@ -42,6 +43,7 @@ func (d *SessionDir) Children() []protocol.File { NewSessionSystemFile(d.sm, d.id), NewSessionThinkingFile(d.sm, d.id), NewSessionPrefillFile(d.sm, d.id), + NewSessionUsageFile(d.sm, d.id), } } @@ -55,6 +57,8 @@ func (d *SessionDir) Lookup(name string) (protocol.File, error) { switch name { case "ask": return NewSessionAskFile(d.sm, d.id), nil + case "compact": + return NewSessionCompactFile(d.sm, d.id), nil case "context": return NewSessionContextFile(d.sm, d.id), nil case "ctl": @@ -69,6 +73,8 @@ func (d *SessionDir) Lookup(name string) (protocol.File, error) { return NewSessionThinkingFile(d.sm, d.id), nil case "prefill": return NewSessionPrefillFile(d.sm, d.id), nil + case "usage": + return NewSessionUsageFile(d.sm, d.id), nil default: return nil, protocol.ErrNotFound } diff --git a/internal/llmfs/session_usage.go b/internal/llmfs/session_usage.go new file mode 100644 index 0000000000000000000000000000000000000000..b80532c72b0d4491019ce47a512403128a433aac --- /dev/null +++ b/internal/llmfs/session_usage.go @@ -0,0 +1,53 @@ +package llmfs + +import ( + "fmt" + "io" + + "github.com/NERVsystems/llm9p/internal/llm" + "github.com/NERVsystems/llm9p/internal/protocol" +) + +// SessionUsageFile provides per-session token usage: /n/llm/N/usage +// Read returns "estimated_tokens/context_limit\n" (e.g., "42000/200000") +// Estimated tokens use a 4 chars/token heuristic over current message content. +type SessionUsageFile struct { + *protocol.BaseFile + sm *llm.SessionManager + id int +} + +// NewSessionUsageFile creates a usage file for the given session. +func NewSessionUsageFile(sm *llm.SessionManager, id int) *SessionUsageFile { + return &SessionUsageFile{ + BaseFile: protocol.NewBaseFile("usage", 0444), + sm: sm, + id: id, + } +} + +// Read returns estimated token usage over context limit. +func (f *SessionUsageFile) Read(p []byte, offset int64) (int, error) { + estimated := f.sm.EstimatedContextTokens(f.id) + limit := f.sm.ContextLimit() + content := fmt.Sprintf("%d/%d\n", estimated, limit) + + if offset >= int64(len(content)) { + return 0, io.EOF + } + return copy(p, content[offset:]), nil +} + +// Write is not allowed. +func (f *SessionUsageFile) Write(p []byte, offset int64) (int, error) { + return 0, fmt.Errorf("usage is read-only") +} + +// Stat returns the file's metadata. +func (f *SessionUsageFile) Stat() protocol.Stat { + s := f.BaseFile.Stat() + estimated := f.sm.EstimatedContextTokens(f.id) + limit := f.sm.ContextLimit() + s.Length = uint64(len(fmt.Sprintf("%d/%d\n", estimated, limit))) + return s +} diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go index ecad15655498ec1a86b834a9f3c0914d7bf3fee8..4973cf16a9e0cdf2dd82ff6fa58519ea394e7f17 100644 --- a/internal/protocol/protocol.go +++ b/internal/protocol/protocol.go @@ -22,8 +22,11 @@ const ( // Version is the protocol version we implement Version = "9P2000" - // MaxMessageSize is the maximum size of a 9P message - MaxMessageSize = 8192 + // MaxMessageSize is the maximum size of a 9P message. + // Must be large enough for system prompts (which can be 10-15KB with + // tool documentation and reminders). Inferno's mount() proposes its + // own msize; the negotiated size is min(client, server). + MaxMessageSize = 65536 // NoTag is used for Tversion/Rversion which don't use tags NoTag uint16 = 0xFFFF