~kris/9p

llm9p

6da61fd486865181bee4d27e82eb163154171892 — pdfinn 6 months ago a3dc06a
feat(llm9p): per-session compact and usage files for context window management

Add automatic context window compaction support to the per-session 9P API:

- session.go: Add Session.EstimatedContextTokens() (4 chars/token heuristic
  over current messages — more accurate than cumulative totalTokens for
  threshold decisions). Add SessionManager.Compact(ctx, id) which summarises
  the conversation via AskWithRequest then replaces session.messages with a
  compact 2-message exchange. Add SessionManager.EstimatedContextTokens(id)
  and SessionManager.ContextLimit() (200K for all Claude models).

- session_compact.go: New /n/llm/N/compact file. Write any content to
  trigger Compact() for that session. Follows the SessionModelFile pattern.

- session_usage.go: New /n/llm/N/usage file. Read returns
  "estimated_tokens/200000\n". Follows the SessionModelFile pattern.

- session_dir.go: Wire compact and usage into Children() and Lookup().

Also land two pre-existing uncommitted fixes:
- cli_client.go: Accept result messages with empty Result field
- protocol.go: Increase MaxMessageSize 8192→65536 for large system prompts

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
M internal/llm/cli_client.go => internal/llm/cli_client.go +1 -1
@@ 387,7 387,7 @@ func parseJSONResponse(output string) (string, error) {
			continue // Not valid JSON, try next line
		}

		if resp.Type == "result" && resp.Result != "" {
		if resp.Type == "result" {
			return resp.Result, nil
		}
	}

M internal/llm/session.go => internal/llm/session.go +84 -0
@@ 4,6 4,8 @@ package llm
import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"sync"
)



@@ 112,6 114,19 @@ func (s *Session) AddTokens(tokens int) {
	s.totalTokens += tokens
}

// EstimatedContextTokens returns a rough token estimate for context window usage.
// Uses 4 chars/token heuristic across all current messages.
// More accurate than totalTokens (which grows quadratically) for threshold decisions.
func (s *Session) EstimatedContextTokens() int {
	s.mu.RLock()
	defer s.mu.RUnlock()
	total := 0
	for _, msg := range s.messages {
		total += len(msg.Content) / 4
	}
	return total
}

// Reset clears the session's conversation history but keeps settings.
func (s *Session) Reset() {
	s.mu.Lock()


@@ 335,6 350,75 @@ func (sm *SessionManager) ListSessions() []int {
	return ids
}

// EstimatedContextTokens returns the estimated token count for a session.
// Delegates to Session.EstimatedContextTokens().
func (sm *SessionManager) EstimatedContextTokens(id int) int {
	session := sm.Get(id)
	if session == nil {
		return 0
	}
	return session.EstimatedContextTokens()
}

// ContextLimit returns the context window limit (200K for all Claude models).
func (sm *SessionManager) ContextLimit() int {
	return 200000
}

// Compact summarizes a session's conversation to reduce context window usage.
// The conversation history is replaced with a compact summary exchange.
// No-op if the session has fewer than 4 messages (nothing meaningful to compact).
func (sm *SessionManager) Compact(ctx context.Context, id int) error {
	session := sm.Get(id)
	if session == nil {
		return ErrSessionNotFound
	}

	session.mu.RLock()
	msgs := make([]Message, len(session.messages))
	copy(msgs, session.messages)
	model := session.model
	session.mu.RUnlock()

	if len(msgs) < 4 {
		return nil
	}

	// Build conversation text for the summarization prompt
	var sb strings.Builder
	for _, msg := range msgs {
		if msg.Role == "system" {
			continue
		}
		sb.WriteString(msg.Role)
		sb.WriteString(": ")
		sb.WriteString(msg.Content)
		sb.WriteString("\n\n")
	}

	req := AskRequest{
		Prompt:      "Summarize this conversation concisely, preserving key facts, decisions, file paths, code snippets, and all context needed to continue the work:\n\n" + sb.String(),
		Model:       model,
		Temperature: 0.3,
	}

	summary, tokens, err := sm.apiClient.AskWithRequest(ctx, req)
	if err != nil {
		return fmt.Errorf("compaction LLM call failed: %w", err)
	}

	// Replace history with a minimal exchange conveying the summary
	session.mu.Lock()
	session.messages = []Message{
		{Role: "user", Content: "Context from earlier in this session:\n" + summary},
		{Role: "assistant", Content: "Understood. I have the context from our previous work and will continue from there."},
	}
	session.totalTokens = tokens
	session.mu.Unlock()

	return nil
}

// AskRequest contains all parameters for an API call.
type AskRequest struct {
	Messages       []Message

A internal/llmfs/session_compact.go => internal/llmfs/session_compact.go +48 -0
@@ 0,0 1,48 @@
package llmfs

import (
	"context"
	"io"

	"github.com/NERVsystems/llm9p/internal/llm"
	"github.com/NERVsystems/llm9p/internal/protocol"
)

// SessionCompactFile triggers conversation compaction for a session: /n/llm/N/compact
// Write any content to trigger; read returns "ok\n" or an error message.
type SessionCompactFile struct {
	*protocol.BaseFile
	sm *llm.SessionManager
	id int
}

// NewSessionCompactFile creates a compact trigger file for the given session.
func NewSessionCompactFile(sm *llm.SessionManager, id int) *SessionCompactFile {
	return &SessionCompactFile{
		BaseFile: protocol.NewBaseFile("compact", 0644),
		sm:       sm,
		id:       id,
	}
}

// Read returns a status line.
func (f *SessionCompactFile) Read(p []byte, offset int64) (int, error) {
	content := "write to compact conversation\n"
	if offset >= int64(len(content)) {
		return 0, io.EOF
	}
	return copy(p, content[offset:]), nil
}

// Write triggers compaction of the session's conversation history.
func (f *SessionCompactFile) Write(p []byte, offset int64) (int, error) {
	if err := f.sm.Compact(context.Background(), f.id); err != nil {
		return 0, protocol.Error("compact: " + err.Error())
	}
	return len(p), nil
}

// Stat returns the file's metadata.
func (f *SessionCompactFile) Stat() protocol.Stat {
	return f.BaseFile.Stat()
}

M internal/llmfs/session_dir.go => internal/llmfs/session_dir.go +7 -1
@@ 10,7 10,7 @@ import (
)

// SessionDir represents a single session directory: /n/llm/N/
// Contains: ask, context, ctl, model, temperature, system, thinking, prefill
// Contains: ask, compact, context, ctl, model, temperature, system, thinking, prefill, usage
type SessionDir struct {
	*protocol.BaseFile
	sm *llm.SessionManager


@@ 35,6 35,7 @@ func (d *SessionDir) Children() []protocol.File {

	return []protocol.File{
		NewSessionAskFile(d.sm, d.id),
		NewSessionCompactFile(d.sm, d.id),
		NewSessionContextFile(d.sm, d.id),
		NewSessionCtlFile(d.sm, d.id),
		NewSessionModelFile(d.sm, d.id),


@@ 42,6 43,7 @@ func (d *SessionDir) Children() []protocol.File {
		NewSessionSystemFile(d.sm, d.id),
		NewSessionThinkingFile(d.sm, d.id),
		NewSessionPrefillFile(d.sm, d.id),
		NewSessionUsageFile(d.sm, d.id),
	}
}



@@ 55,6 57,8 @@ func (d *SessionDir) Lookup(name string) (protocol.File, error) {
	switch name {
	case "ask":
		return NewSessionAskFile(d.sm, d.id), nil
	case "compact":
		return NewSessionCompactFile(d.sm, d.id), nil
	case "context":
		return NewSessionContextFile(d.sm, d.id), nil
	case "ctl":


@@ 69,6 73,8 @@ func (d *SessionDir) Lookup(name string) (protocol.File, error) {
		return NewSessionThinkingFile(d.sm, d.id), nil
	case "prefill":
		return NewSessionPrefillFile(d.sm, d.id), nil
	case "usage":
		return NewSessionUsageFile(d.sm, d.id), nil
	default:
		return nil, protocol.ErrNotFound
	}

A internal/llmfs/session_usage.go => internal/llmfs/session_usage.go +53 -0
@@ 0,0 1,53 @@
package llmfs

import (
	"fmt"
	"io"

	"github.com/NERVsystems/llm9p/internal/llm"
	"github.com/NERVsystems/llm9p/internal/protocol"
)

// SessionUsageFile provides per-session token usage: /n/llm/N/usage
// Read returns "estimated_tokens/context_limit\n" (e.g., "42000/200000")
// Estimated tokens use a 4 chars/token heuristic over current message content.
type SessionUsageFile struct {
	*protocol.BaseFile
	sm *llm.SessionManager
	id int
}

// NewSessionUsageFile creates a usage file for the given session.
func NewSessionUsageFile(sm *llm.SessionManager, id int) *SessionUsageFile {
	return &SessionUsageFile{
		BaseFile: protocol.NewBaseFile("usage", 0444),
		sm:       sm,
		id:       id,
	}
}

// Read returns estimated token usage over context limit.
func (f *SessionUsageFile) Read(p []byte, offset int64) (int, error) {
	estimated := f.sm.EstimatedContextTokens(f.id)
	limit := f.sm.ContextLimit()
	content := fmt.Sprintf("%d/%d\n", estimated, limit)

	if offset >= int64(len(content)) {
		return 0, io.EOF
	}
	return copy(p, content[offset:]), nil
}

// Write is not allowed.
func (f *SessionUsageFile) Write(p []byte, offset int64) (int, error) {
	return 0, fmt.Errorf("usage is read-only")
}

// Stat returns the file's metadata.
func (f *SessionUsageFile) Stat() protocol.Stat {
	s := f.BaseFile.Stat()
	estimated := f.sm.EstimatedContextTokens(f.id)
	limit := f.sm.ContextLimit()
	s.Length = uint64(len(fmt.Sprintf("%d/%d\n", estimated, limit)))
	return s
}

M internal/protocol/protocol.go => internal/protocol/protocol.go +5 -2
@@ 22,8 22,11 @@ const (
	// Version is the protocol version we implement
	Version = "9P2000"

	// MaxMessageSize is the maximum size of a 9P message
	MaxMessageSize = 8192
	// MaxMessageSize is the maximum size of a 9P message.
	// Must be large enough for system prompts (which can be 10-15KB with
	// tool documentation and reminders). Inferno's mount() proposes its
	// own msize; the negotiated size is min(client, server).
	MaxMessageSize = 65536

	// NoTag is used for Tversion/Rversion which don't use tags
	NoTag uint16 = 0xFFFF