Add memory system, dynamic tool loading, and fix logging issues

- Add MemoryStore for persistent long-term and daily notes - Add dynamic tool summary generation in system prompt - Fix YAML frontmatter parsing for nanobot skill format - Add GetSummaries() method to ToolRegistry - Fix DebugCF logging to use structured metadata - Improve web_search and shell tool descriptions
2026-02-10 23:33:28 +08:00
parent 10442732b4
commit 21d60f63fc
8 changed files with 341 additions and 395 deletions
--- a/pkg/agent/context.go
+++ b/pkg/agent/context.go
@@ -17,14 +17,29 @@ type ContextBuilder struct {
 	workspace    string
 	skillsLoader *skills.SkillsLoader
 	memory       *MemoryStore
+	toolsSummary func() []string // Function to get tool summaries dynamically
 }

-func NewContextBuilder(workspace string) *ContextBuilder {
-	builtinSkillsDir := filepath.Join(filepath.Dir(workspace), "picoclaw", "skills")
+func getGlobalConfigDir() string {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return ""
+	}
+	return filepath.Join(home, ".picoclaw")
+}
+
+func NewContextBuilder(workspace string, toolsSummaryFunc func() []string) *ContextBuilder {
+	// builtin skills: 当前项目的 skills 目录
+	// 使用当前工作目录下的 skills/ 目录
+	wd, _ := os.Getwd()
+	builtinSkillsDir := filepath.Join(wd, "skills")
+	globalSkillsDir := filepath.Join(getGlobalConfigDir(), "skills")
+
 	return &ContextBuilder{
 		workspace:    workspace,
-		skillsLoader: skills.NewSkillsLoader(workspace, builtinSkillsDir),
+		skillsLoader: skills.NewSkillsLoader(workspace, globalSkillsDir, builtinSkillsDir),
 		memory:       NewMemoryStore(workspace),
+		toolsSummary: toolsSummaryFunc,
 	}
 }

@@ -33,14 +48,12 @@ func (cb *ContextBuilder) getIdentity() string {
 	workspacePath, _ := filepath.Abs(filepath.Join(cb.workspace))
 	runtime := fmt.Sprintf("%s %s, Go %s", runtime.GOOS, runtime.GOARCH, runtime.Version())

+	// Build tools section dynamically
+	toolsSection := cb.buildToolsSection()
+
 	return fmt.Sprintf(`# picoclaw 🦞

-You are picoclaw, a helpful AI assistant. You have access to tools that allow you to:
- Read, write, and edit files
- Execute shell commands
- Search the web and fetch web pages
- Send messages to users on chat channels
- Spawn subagents for complex background tasks
+You are picoclaw, a helpful AI assistant.

 ## Current Time
 %s
@@ -50,26 +63,36 @@ You are picoclaw, a helpful AI assistant. You have access to tools that allow yo

 ## Workspace
 Your workspace is at: %s
- Memory files: %s/memory/MEMORY.md
- Daily notes: %s/memory/2006-01-02.md
- Custom skills: %s/skills/{skill-name}/SKILL.md
+- Memory: %s/memory/MEMORY.md
+- Daily Notes: %s/memory/YYYYMM/YYYYMMDD.md
+- Skills: %s/skills/{skill-name}/SKILL.md

-## Weather Information
-When users ask about weather, use the web_fetch tool with wttr.in URLs:
- Current weather: https://wttr.in/{city}?format=j1
- Beijing: https://wttr.in/Beijing?format=j1
- Shanghai: https://wttr.in/Shanghai?format=j1
- New York: https://wttr.in/New_York?format=j1
- London: https://wttr.in/London?format=j1
- Tokyo: https://wttr.in/Tokyo?format=j1
-
-IMPORTANT: When responding to direct questions or conversations, reply directly with your text response.
-Only use the 'message' tool when you need to send a message to a specific chat channel (like WhatsApp).
-For normal conversation, just respond with text - do not call the message tool.
+%s

 Always be helpful, accurate, and concise. When using tools, explain what you're doing.
 When remembering something, write to %s/memory/MEMORY.md`,
-		now, runtime, workspacePath, workspacePath, workspacePath, workspacePath, workspacePath)
+		now, runtime, workspacePath, workspacePath, workspacePath, workspacePath, toolsSection, workspacePath)
+}
+
+func (cb *ContextBuilder) buildToolsSection() string {
+	if cb.toolsSummary == nil {
+		return ""
+	}
+
+	summaries := cb.toolsSummary()
+	if len(summaries) == 0 {
+		return ""
+	}
+
+	var sb strings.Builder
+	sb.WriteString("## Available Tools\n\n")
+	sb.WriteString("You have access to the following tools:\n\n")
+	for _, s := range summaries {
+		sb.WriteString(s)
+		sb.WriteString("\n")
+	}
+
+	return sb.String()
 }

 func (cb *ContextBuilder) BuildSystemPrompt() string {
@@ -84,22 +107,12 @@ func (cb *ContextBuilder) BuildSystemPrompt() string {
 		parts = append(parts, bootstrapContent)
 	}

-	// Skills - progressive loading
-	// 1. Always skills: load full content
-	alwaysSkills := cb.skillsLoader.GetAlwaysSkills()
-	if len(alwaysSkills) > 0 {
-		alwaysContent := cb.skillsLoader.LoadSkillsForContext(alwaysSkills)
-		if alwaysContent != "" {
-			parts = append(parts, "# Active Skills\n\n"+alwaysContent)
-		}
-	}
-
-	// 2. Available skills: only show summary
+	// Skills - show summary, AI can read full content with read_file tool
 	skillsSummary := cb.skillsLoader.BuildSkillsSummary()
 	if skillsSummary != "" {
 		parts = append(parts, fmt.Sprintf(`# Skills

-The following skills extend your capabilities. To use a skill, read its SKILL.md file.
+The following skills extend your capabilities. To use a skill, read its SKILL.md file using the read_file tool.

 %s`, skillsSummary))
 	}
@@ -119,9 +132,7 @@ func (cb *ContextBuilder) LoadBootstrapFiles() string {
 		"AGENTS.md",
 		"SOUL.md",
 		"USER.md",
-		"TOOLS.md",
 		"IDENTITY.md",
-		"MEMORY.md",
 	}

 	var result string
@@ -149,14 +160,23 @@ func (cb *ContextBuilder) BuildMessages(history []providers.Message, summary str
 		systemPrompt += fmt.Sprintf("\n\n## Current Session\nChannel: %s\nChat ID: %s", channel, chatID)
 	}

-	// Log system prompt for debugging
-	logger.InfoCF("agent", "System prompt built",
+	// Log system prompt summary for debugging (debug mode only)
+	logger.DebugCF("agent", "System prompt built",
 		map[string]interface{}{
 			"total_chars": len(systemPrompt),
 			"total_lines": strings.Count(systemPrompt, "\n") + 1,
 			"section_count": strings.Count(systemPrompt, "\n\n---\n\n") + 1,
 		})
-	logger.DebugCF("agent", "Full system prompt:\n"+systemPrompt, nil)
+
+	// Log preview of system prompt (avoid logging huge content)
+	preview := systemPrompt
+	if len(preview) > 500 {
+		preview = preview[:500] + "... (truncated)"
+	}
+	logger.DebugCF("agent", "System prompt preview",
+		map[string]interface{}{
+			"preview": preview,
+		})

 	if summary != "" {
 		systemPrompt += "\n\n## Summary of Previous Conversation\n\n" + summary
@@ -191,14 +211,13 @@ func (cb *ContextBuilder) AddAssistantMessage(messages []providers.Message, cont
 		Role:    "assistant",
 		Content: content,
 	}
-	if len(toolCalls) > 0 {
-		messages = append(messages, msg)
-	}
+	// Always add assistant message, whether or not it has tool calls
+	messages = append(messages, msg)
 	return messages
 }

 func (cb *ContextBuilder) loadSkills() string {
-	allSkills := cb.skillsLoader.ListSkills(true)
+	allSkills := cb.skillsLoader.ListSkills()
 	if len(allSkills) == 0 {
 		return ""
 	}
@@ -218,18 +237,13 @@ func (cb *ContextBuilder) loadSkills() string {

 // GetSkillsInfo returns information about loaded skills.
 func (cb *ContextBuilder) GetSkillsInfo() map[string]interface{} {
-	allSkills := cb.skillsLoader.ListSkills(true)
+	allSkills := cb.skillsLoader.ListSkills()
 	skillNames := make([]string, 0, len(allSkills))
-	availableCount := 0
 	for _, s := range allSkills {
 		skillNames = append(skillNames, s.Name)
-		if s.Available {
-			availableCount++
-		}
 	}
 	return map[string]interface{}{
-		"total":     len(allSkills),
-		"available": availableCount,
-		"names":     skillNames,
+		"total": len(allSkills),
+		"names": skillNames,
 	}
 }
--- a/pkg/agent/loop.go
+++ b/pkg/agent/loop.go
@@ -78,7 +78,7 @@ func NewAgentLoop(cfg *config.Config, msgBus *bus.MessageBus, provider providers
 		model:          cfg.Agents.Defaults.Model,
 		maxIterations:  cfg.Agents.Defaults.MaxToolIterations,
 		sessions:       sessionsManager,
-		contextBuilder: NewContextBuilder(workspace),
+		contextBuilder: NewContextBuilder(workspace, func() []string { return toolsRegistry.GetSummaries() }),
 		tools:          toolsRegistry,
 		running:        false,
 	}
@@ -159,12 +159,8 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage)
 		}
 	}

-	history := al.sessions.GetHistory(msg.SessionKey)
-	summary := al.sessions.GetSummary(msg.SessionKey)
-
 	messages := al.contextBuilder.BuildMessages(
-		history,
-		summary,
+		al.sessions.GetHistory(msg.SessionKey),
 		msg.Content,
 		nil,
 		msg.Channel,
@@ -196,6 +192,26 @@ func (al *AgentLoop) processMessage(ctx context.Context, msg bus.InboundMessage)
 			})
 		}

+		// Log LLM request details
+		logger.DebugCF("agent", "LLM request",
+			map[string]interface{}{
+				"iteration":        iteration,
+				"model":            al.model,
+				"messages_count":   len(messages),
+				"tools_count":      len(providerToolDefs),
+				"max_tokens":       8192,
+				"temperature":      0.7,
+				"system_prompt_len": len(messages[0].Content),
+			})
+
+		// Log full messages (detailed)
+		logger.DebugCF("agent", "Full LLM request",
+			map[string]interface{}{
+				"iteration":     iteration,
+				"messages_json": formatMessagesForLog(messages),
+				"tools_json":    formatToolsForLog(providerToolDefs),
+			})
+
 		response, err := al.provider.Chat(ctx, messages, providerToolDefs, al.model, map[string]interface{}{
 			"max_tokens":  8192,
 			"temperature": 0.7,
@@ -331,11 +347,8 @@ func (al *AgentLoop) processSystemMessage(ctx context.Context, msg bus.InboundMe
 	}

 	// Build messages with the announce content
-	history := al.sessions.GetHistory(sessionKey)
-	summary := al.sessions.GetSummary(sessionKey)
 	messages := al.contextBuilder.BuildMessages(
-		history,
-		summary,
+		al.sessions.GetHistory(sessionKey),
 		msg.Content,
 		nil,
 		originChannel,
@@ -361,6 +374,26 @@ func (al *AgentLoop) processSystemMessage(ctx context.Context, msg bus.InboundMe
 			})
 		}

+		// Log LLM request details
+		logger.DebugCF("agent", "LLM request",
+			map[string]interface{}{
+				"iteration":        iteration,
+				"model":            al.model,
+				"messages_count":   len(messages),
+				"tools_count":      len(providerToolDefs),
+				"max_tokens":       8192,
+				"temperature":      0.7,
+				"system_prompt_len": len(messages[0].Content),
+			})
+
+		// Log full messages (detailed)
+		logger.DebugCF("agent", "Full LLM request",
+			map[string]interface{}{
+				"iteration":     iteration,
+				"messages_json": formatMessagesForLog(messages),
+				"tools_json":    formatToolsForLog(providerToolDefs),
+			})
+
 		response, err := al.provider.Chat(ctx, messages, providerToolDefs, al.model, map[string]interface{}{
 			"max_tokens":  8192,
 			"temperature": 0.7,
@@ -462,104 +495,64 @@ func (al *AgentLoop) GetStartupInfo() map[string]interface{} {
 	return info
 }

-func (al *AgentLoop) summarizeSession(sessionKey string) {
-	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
-	defer cancel()
-
-	history := al.sessions.GetHistory(sessionKey)
-	summary := al.sessions.GetSummary(sessionKey)
-
-	// Keep last 4 messages for continuity
-	if len(history) <= 4 {
-		return
+// formatMessagesForLog formats messages for logging
+func formatMessagesForLog(messages []providers.Message) string {
+	if len(messages) == 0 {
+		return "[]"
 	}

-	toSummarize := history[:len(history)-4]
-
-	// Oversized Message Guard (Dynamic)
-	// Skip messages larger than 50% of context window to prevent summarizer overflow.
-	maxMessageTokens := al.contextWindow / 2
-	validMessages := make([]providers.Message, 0)
-	omitted := false
-
-	for _, m := range toSummarize {
-		if m.Role != "user" && m.Role != "assistant" {
-			continue
+	var result string
+	result += "[\n"
+	for i, msg := range messages {
+		result += fmt.Sprintf("  [%d] Role: %s\n", i, msg.Role)
+		if msg.ToolCalls != nil && len(msg.ToolCalls) > 0 {
+			result += "  ToolCalls:\n"
+			for _, tc := range msg.ToolCalls {
+				result += fmt.Sprintf("    - ID: %s, Type: %s, Name: %s\n", tc.ID, tc.Type, tc.Name)
+				if tc.Function != nil {
+					result += fmt.Sprintf("      Arguments: %s\n", truncateString(tc.Function.Arguments, 200))
+				}
+			}
 		}
-		// Estimate tokens for this message
-		msgTokens := len(m.Content) / 4
-		if msgTokens > maxMessageTokens {
-			omitted = true
-			continue
+		if msg.Content != "" {
+			content := truncateString(msg.Content, 200)
+			result += fmt.Sprintf("  Content: %s\n", content)
 		}
-		validMessages = append(validMessages, m)
-	}
-
-	if len(validMessages) == 0 {
-		return
-	}
-
-	// Multi-Part Summarization
-	// Split into two parts if history is significant
-	var finalSummary string
-	if len(validMessages) > 10 {
-		mid := len(validMessages) / 2
-		part1 := validMessages[:mid]
-		part2 := validMessages[mid:]
-
-		s1, _ := al.summarizeBatch(ctx, part1, "")
-		s2, _ := al.summarizeBatch(ctx, part2, "")
-		
-		// Merge them
-		mergePrompt := fmt.Sprintf("Merge these two conversation summaries into one cohesive summary:\n\n1: %s\n\n2: %s", s1, s2)
-		resp, err := al.provider.Chat(ctx, []providers.Message{{Role: "user", Content: mergePrompt}}, nil, al.model, map[string]interface{}{
-			"max_tokens":  1024,
-			"temperature": 0.3,
-		})
-		if err == nil {
-			finalSummary = resp.Content
-		} else {
-			finalSummary = s1 + " " + s2
+		if msg.ToolCallID != "" {
+			result += fmt.Sprintf("  ToolCallID: %s\n", msg.ToolCallID)
 		}
-	} else {
-		finalSummary, _ = al.summarizeBatch(ctx, validMessages, summary)
-	}
-
-	if omitted && finalSummary != "" {
-		finalSummary += "\n[Note: Some oversized messages were omitted from this summary for efficiency.]"
-	}
-
-	if finalSummary != "" {
-		al.sessions.SetSummary(sessionKey, finalSummary)
-		al.sessions.TruncateHistory(sessionKey, 4)
-		al.sessions.Save(al.sessions.GetOrCreate(sessionKey))
+		result += "\n"
 	}
+	result += "]"
+	return result
 }

-func (al *AgentLoop) summarizeBatch(ctx context.Context, batch []providers.Message, existingSummary string) (string, error) {
-	prompt := "Provide a concise summary of this conversation segment, preserving core context and key points.\n"
-	if existingSummary != "" {
-		prompt += "Existing context: " + existingSummary + "\n"
-	}
-	prompt += "\nCONVERSATION:\n"
-	for _, m := range batch {
-		prompt += fmt.Sprintf("%s: %s\n", m.Role, m.Content)
+// formatToolsForLog formats tool definitions for logging
+func formatToolsForLog(tools []providers.ToolDefinition) string {
+	if len(tools) == 0 {
+		return "[]"
 	}

-	response, err := al.provider.Chat(ctx, []providers.Message{{Role: "user", Content: prompt}}, nil, al.model, map[string]interface{}{
-		"max_tokens":  1024,
-		"temperature": 0.3,
-	})
-	if err != nil {
-		return "", err
+	var result string
+	result += "[\n"
+	for i, tool := range tools {
+		result += fmt.Sprintf("  [%d] Type: %s, Name: %s\n", i, tool.Type, tool.Function.Name)
+		result += fmt.Sprintf("      Description: %s\n", tool.Function.Description)
+		if len(tool.Function.Parameters) > 0 {
+			result += fmt.Sprintf("      Parameters: %s\n", truncateString(fmt.Sprintf("%v", tool.Function.Parameters), 200))
+		}
 	}
-	return response.Content, nil
+	result += "]"
+	return result
 }

-func (al *AgentLoop) estimateTokens(messages []providers.Message) int {
-	total := 0
-	for _, m := range messages {
-		total += len(m.Content) / 4 // Simple heuristic: 4 chars per token
+// truncateString truncates a string to max length
+func truncateString(s string, maxLen int) string {
+	if len(s) <= maxLen {
+		return s
 	}
-	return total
+	if maxLen <= 3 {
+		return s[:maxLen]
+	}
+	return s[:maxLen-3] + "..."
 }
--- a/pkg/agent/memory.go
+++ b/pkg/agent/memory.go
@@ -10,12 +10,12 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 	"time"
 )

 // MemoryStore manages persistent memory for the agent.
-// Supports daily notes (memory/YYYY-MM-DD.md) and long-term memory (MEMORY.md).
+// - Long-term memory: memory/MEMORY.md
+// - Daily notes: memory/YYYYMM/YYYYMMDD.md
 type MemoryStore struct {
 	workspace  string
 	memoryDir  string
@@ -38,23 +38,29 @@ func NewMemoryStore(workspace string) *MemoryStore {
 	}
 }

-// getMemoryDir returns the memory directory path.
-func (ms *MemoryStore) getMemoryDir() string {
-	return ms.memoryDir
-}
-
-// getMemoryFile returns the long-term memory file path.
-func (ms *MemoryStore) getMemoryFile() string {
-	return ms.memoryFile
-}
-
-// getTodayFile returns the path to today's memory file (YYYY-MM-DD.md).
+// getTodayFile returns the path to today's daily note file (memory/YYYYMM/YYYYMMDD.md).
 func (ms *MemoryStore) getTodayFile() string {
-	today := time.Now().Format("2006-01-02")
-	return filepath.Join(ms.memoryDir, today+".md")
+	today := time.Now().Format("20060102")      // YYYYMMDD
+	monthDir := today[:6]                       // YYYYMM
+	filePath := filepath.Join(ms.memoryDir, monthDir, today+".md")
+	return filePath
 }

-// ReadToday reads today's memory notes.
+// ReadLongTerm reads the long-term memory (MEMORY.md).
+// Returns empty string if the file doesn't exist.
+func (ms *MemoryStore) ReadLongTerm() string {
+	if data, err := os.ReadFile(ms.memoryFile); err == nil {
+		return string(data)
+	}
+	return ""
+}
+
+// WriteLongTerm writes content to the long-term memory file (MEMORY.md).
+func (ms *MemoryStore) WriteLongTerm(content string) error {
+	return os.WriteFile(ms.memoryFile, []byte(content), 0644)
+}
+
+// ReadToday reads today's daily note.
 // Returns empty string if the file doesn't exist.
 func (ms *MemoryStore) ReadToday() string {
 	todayFile := ms.getTodayFile()
@@ -64,11 +70,15 @@ func (ms *MemoryStore) ReadToday() string {
 	return ""
 }

-// AppendToday appends content to today's memory notes.
+// AppendToday appends content to today's daily note.
 // If the file doesn't exist, it creates a new file with a date header.
 func (ms *MemoryStore) AppendToday(content string) error {
 	todayFile := ms.getTodayFile()

+	// Ensure month directory exists
+	monthDir := filepath.Dir(todayFile)
+	os.MkdirAll(monthDir, 0755)
+
 	var existingContent string
 	if data, err := os.ReadFile(todayFile); err == nil {
 		existingContent = string(data)
@@ -87,46 +97,39 @@ func (ms *MemoryStore) AppendToday(content string) error {
 	return os.WriteFile(todayFile, []byte(newContent), 0644)
 }

-// ReadLongTerm reads the long-term memory (MEMORY.md).
-// Returns empty string if the file doesn't exist.
-func (ms *MemoryStore) ReadLongTerm() string {
-	if data, err := os.ReadFile(ms.memoryFile); err == nil {
-		return string(data)
-	}
-	return ""
-}
-
-// WriteLongTerm writes content to the long-term memory file (MEMORY.md).
-func (ms *MemoryStore) WriteLongTerm(content string) error {
-	return os.WriteFile(ms.memoryFile, []byte(content), 0644)
-}
-
-// GetRecentMemories returns memories from the last N days.
-// It reads and combines the contents of memory files from the past days.
+// GetRecentDailyNotes returns daily notes from the last N days.
 // Contents are joined with "---" separator.
-func (ms *MemoryStore) GetRecentMemories(days int) string {
-	var memories []string
+func (ms *MemoryStore) GetRecentDailyNotes(days int) string {
+	var notes []string

 	for i := 0; i < days; i++ {
 		date := time.Now().AddDate(0, 0, -i)
-		dateStr := date.Format("2006-01-02")
-		filePath := filepath.Join(ms.memoryDir, dateStr+".md")
+		dateStr := date.Format("20060102")      // YYYYMMDD
+		monthDir := dateStr[:6]                 // YYYYMM
+		filePath := filepath.Join(ms.memoryDir, monthDir, dateStr+".md")

 		if data, err := os.ReadFile(filePath); err == nil {
-			memories = append(memories, string(data))
+			notes = append(notes, string(data))
 		}
 	}

-	if len(memories) == 0 {
+	if len(notes) == 0 {
 		return ""
 	}

-	return strings.Join(memories, "\n\n---\n\n")
+	// Join with separator
+	var result string
+	for i, note := range notes {
+		if i > 0 {
+			result += "\n\n---\n\n"
+		}
+		result += note
+	}
+	return result
 }

 // GetMemoryContext returns formatted memory context for the agent prompt.
-// It includes long-term memory and today's notes sections if they exist.
-// Returns empty string if no memory exists.
+// Includes long-term memory and recent daily notes.
 func (ms *MemoryStore) GetMemoryContext() string {
 	var parts []string

@@ -136,15 +139,23 @@ func (ms *MemoryStore) GetMemoryContext() string {
 		parts = append(parts, "## Long-term Memory\n\n"+longTerm)
 	}

-	// Today's notes
-	today := ms.ReadToday()
-	if today != "" {
-		parts = append(parts, "## Today's Notes\n\n"+today)
+	// Recent daily notes (last 3 days)
+	recentNotes := ms.GetRecentDailyNotes(3)
+	if recentNotes != "" {
+		parts = append(parts, "## Recent Daily Notes\n\n"+recentNotes)
 	}

 	if len(parts) == 0 {
 		return ""
 	}

-	return strings.Join(parts, "\n\n")
+	// Join parts with separator
+	var result string
+	for i, part := range parts {
+		if i > 0 {
+			result += "\n\n---\n\n"
+		}
+		result += part
+	}
+	return fmt.Sprintf("# Memory\n\n%s", result)
 }