From 9936dbce52b9c2272457a24cb6fe096531397207 Mon Sep 17 00:00:00 2001 From: lxowalle Date: Tue, 10 Feb 2026 17:10:41 +0800 Subject: [PATCH] * Discord & Telegram support ASR through groq --- README.md | 2 +- cmd/picoclaw/main.go | 21 ++++--- go.mod | 8 +-- go.sum | 12 ++-- pkg/channels/discord.go | 120 +++++++++++++++++++++++++++++++++++++-- pkg/channels/telegram.go | 60 ++++++++++++++++++-- pkg/voice/transcriber.go | 61 ++++++++++++++++++-- 7 files changed, 250 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 279845f..929b364 100644 --- a/README.md +++ b/README.md @@ -278,7 +278,7 @@ Config file: `~/.picoclaw/config.json` | `anthropic(To be tested)` | LLM (Claude direct) | [console.anthropic.com](https://console.anthropic.com) | | `openai(To be tested)` | LLM (GPT direct) | [platform.openai.com](https://platform.openai.com) | | `deepseek(To be tested)` | LLM (DeepSeek direct) | [platform.deepseek.com](https://platform.deepseek.com) | -| `groq(To be tested)` | LLM + **Voice transcription** (Whisper) | [console.groq.com](https://console.groq.com) | +| `groq` | LLM + **Voice transcription** (Whisper) | [console.groq.com](https://console.groq.com) |
diff --git a/cmd/picoclaw/main.go b/cmd/picoclaw/main.go index cdec201..23bb7b9 100644 --- a/cmd/picoclaw/main.go +++ b/cmd/picoclaw/main.go @@ -142,7 +142,8 @@ func main() { func printHelp() { fmt.Printf("%s picoclaw - Personal AI Assistant v%s\n\n", logo, version) - fmt.Println("Usage: picoclaw \n") + fmt.Println("Usage: picoclaw ") + fmt.Println() fmt.Println("Commands:") fmt.Println(" onboard Initialize picoclaw configuration and workspace") fmt.Println(" agent Interact with the agent directly") @@ -450,8 +451,8 @@ func agentCmd() { os.Exit(1) } - bus := bus.NewMessageBus() - agentLoop := agent.NewAgentLoop(cfg, bus, provider) + msgBus := bus.NewMessageBus() + agentLoop := agent.NewAgentLoop(cfg, msgBus, provider) if message != "" { ctx := context.Background() @@ -472,7 +473,7 @@ func interactiveMode(agentLoop *agent.AgentLoop, sessionKey string) { rl, err := readline.NewEx(&readline.Config{ Prompt: prompt, - HistoryFile: "/tmp/.picoclaw_history", + HistoryFile: filepath.Join(os.TempDir(), ".picoclaw_history"), HistoryLimit: 100, InterruptPrompt: "^C", EOFPrompt: "exit", @@ -566,8 +567,8 @@ func gatewayCmd() { os.Exit(1) } - bus := bus.NewMessageBus() - agentLoop := agent.NewAgentLoop(cfg, bus, provider) + msgBus := bus.NewMessageBus() + agentLoop := agent.NewAgentLoop(cfg, msgBus, provider) cronStorePath := filepath.Join(filepath.Dir(getConfigPath()), "cron", "jobs.json") cronService := cron.NewCronService(cronStorePath, nil) @@ -579,7 +580,7 @@ func gatewayCmd() { true, ) - channelManager, err := channels.NewManager(cfg, bus) + channelManager, err := channels.NewManager(cfg, msgBus) if err != nil { fmt.Printf("Error creating channel manager: %v\n", err) os.Exit(1) @@ -598,6 +599,12 @@ func gatewayCmd() { logger.InfoC("voice", "Groq transcription attached to Telegram channel") } } + if discordChannel, ok := channelManager.GetChannel("discord"); ok { + if dc, ok := discordChannel.(*channels.DiscordChannel); ok { + dc.SetTranscriber(transcriber) + logger.InfoC("voice", "Groq transcription attached to Discord channel") + } + } } enabledChannels := channelManager.GetEnabledChannels() diff --git a/go.mod b/go.mod index 9490f79..d07ae18 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,9 @@ module github.com/sipeed/picoclaw -go 1.18 +go 1.24.0 require ( - github.com/bwmarrin/discordgo v0.28.1 + github.com/bwmarrin/discordgo v0.29.0 github.com/caarlos0/env/v11 v11.3.1 github.com/chzyer/readline v1.5.1 github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1 @@ -13,6 +13,6 @@ require ( require ( github.com/gogo/protobuf v1.3.2 // indirect - golang.org/x/crypto v0.28.0 // indirect - golang.org/x/sys v0.40.0 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/sys v0.41.0 // indirect ) diff --git a/go.sum b/go.sum index f3a1c57..acbad21 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/bwmarrin/discordgo v0.28.1 h1:gXsuo2GBO7NbR6uqmrrBDplPUx2T3nzu775q/Rd1aG4= -github.com/bwmarrin/discordgo v0.28.1/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY= +github.com/bwmarrin/discordgo v0.29.0 h1:FmWeXFaKUwrcL3Cx65c20bTRW+vOb6k8AnaP+EgjDno= +github.com/bwmarrin/discordgo v0.29.0/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY= github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA= github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= @@ -26,8 +26,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -43,8 +43,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/pkg/channels/discord.go b/pkg/channels/discord.go index 12df5e9..ba455f0 100644 --- a/pkg/channels/discord.go +++ b/pkg/channels/discord.go @@ -3,17 +3,26 @@ package channels import ( "context" "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "time" "github.com/bwmarrin/discordgo" "github.com/sipeed/picoclaw/pkg/bus" "github.com/sipeed/picoclaw/pkg/config" "github.com/sipeed/picoclaw/pkg/logger" + "github.com/sipeed/picoclaw/pkg/voice" ) type DiscordChannel struct { *BaseChannel - session *discordgo.Session - config config.DiscordConfig + session *discordgo.Session + config config.DiscordConfig + transcriber *voice.GroqTranscriber } func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordChannel, error) { @@ -28,9 +37,14 @@ func NewDiscordChannel(cfg config.DiscordConfig, bus *bus.MessageBus) (*DiscordC BaseChannel: base, session: session, config: cfg, + transcriber: nil, }, nil } +func (c *DiscordChannel) SetTranscriber(transcriber *voice.GroqTranscriber) { + c.transcriber = transcriber +} + func (c *DiscordChannel) Start(ctx context.Context) error { logger.InfoC("discord", "Starting Discord bot") @@ -103,11 +117,48 @@ func (c *DiscordChannel) handleMessage(s *discordgo.Session, m *discordgo.Messag mediaPaths := []string{} for _, attachment := range m.Attachments { - mediaPaths = append(mediaPaths, attachment.URL) - if content != "" { - content += "\n" + isAudio := isAudioFile(attachment.Filename, attachment.ContentType) + + if isAudio { + localPath := c.downloadAttachment(attachment.URL, attachment.Filename) + if localPath != "" { + mediaPaths = append(mediaPaths, localPath) + + transcribedText := "" + if c.transcriber != nil && c.transcriber.IsAvailable() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + result, err := c.transcriber.Transcribe(ctx, localPath) + if err != nil { + log.Printf("Voice transcription failed: %v", err) + transcribedText = fmt.Sprintf("[audio: %s (transcription failed)]", localPath) + } else { + transcribedText = fmt.Sprintf("[audio transcription: %s]", result.Text) + log.Printf("Audio transcribed successfully: %s", result.Text) + } + } else { + transcribedText = fmt.Sprintf("[audio: %s]", localPath) + } + + if content != "" { + content += "\n" + } + content += transcribedText + } else { + mediaPaths = append(mediaPaths, attachment.URL) + if content != "" { + content += "\n" + } + content += fmt.Sprintf("[attachment: %s]", attachment.URL) + } + } else { + mediaPaths = append(mediaPaths, attachment.URL) + if content != "" { + content += "\n" + } + content += fmt.Sprintf("[attachment: %s]", attachment.URL) } - content += fmt.Sprintf("[attachment: %s]", attachment.URL) } if content == "" && len(mediaPaths) == 0 { @@ -136,3 +187,60 @@ func (c *DiscordChannel) handleMessage(s *discordgo.Session, m *discordgo.Messag c.HandleMessage(senderID, m.ChannelID, content, mediaPaths, metadata) } + +func isAudioFile(filename, contentType string) bool { + audioExtensions := []string{".mp3", ".wav", ".ogg", ".m4a", ".flac", ".aac", ".wma"} + audioTypes := []string{"audio/", "application/ogg", "application/x-ogg"} + + for _, ext := range audioExtensions { + if strings.HasSuffix(strings.ToLower(filename), ext) { + return true + } + } + + for _, audioType := range audioTypes { + if strings.HasPrefix(strings.ToLower(contentType), audioType) { + return true + } + } + + return false +} + +func (c *DiscordChannel) downloadAttachment(url, filename string) string { + mediaDir := filepath.Join(os.TempDir(), "picoclaw_media") + if err := os.MkdirAll(mediaDir, 0755); err != nil { + log.Printf("Failed to create media directory: %v", err) + return "" + } + + localPath := filepath.Join(mediaDir, filename) + + resp, err := http.Get(url) + if err != nil { + log.Printf("Failed to download attachment: %v", err) + return "" + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Printf("Failed to download attachment, status: %d", resp.StatusCode) + return "" + } + + out, err := os.Create(localPath) + if err != nil { + log.Printf("Failed to create file: %v", err) + return "" + } + defer out.Close() + + _, err = io.Copy(out, resp.Body) + if err != nil { + log.Printf("Failed to write file: %v", err) + return "" + } + + log.Printf("Attachment downloaded successfully to: %s", localPath) + return localPath +} diff --git a/pkg/channels/telegram.go b/pkg/channels/telegram.go index 1ad41f9..2a14127 100644 --- a/pkg/channels/telegram.go +++ b/pkg/channels/telegram.go @@ -3,7 +3,11 @@ package channels import ( "context" "fmt" + "io" "log" + "net/http" + "os" + "path/filepath" "regexp" "strings" "sync" @@ -305,9 +309,20 @@ func (c *TelegramChannel) downloadFileWithInfo(file *tgbotapi.File, ext string) url := file.Link(c.bot.Token) log.Printf("File URL: %s", url) - mediaDir := "/tmp/picoclaw_media" + mediaDir := filepath.Join(os.TempDir(), "picoclaw_media") + if err := os.MkdirAll(mediaDir, 0755); err != nil { + log.Printf("Failed to create media directory: %v", err) + return "" + } - return fmt.Sprintf("%s/%s%s", mediaDir, file.FilePath[:min(16, len(file.FilePath))], ext) + localPath := filepath.Join(mediaDir, file.FilePath[:min(16, len(file.FilePath))]+ext) + + if err := c.downloadFromURL(url, localPath); err != nil { + log.Printf("Failed to download file: %v", err) + return "" + } + + return localPath } func min(a, b int) int { @@ -317,6 +332,32 @@ func min(a, b int) int { return b } +func (c *TelegramChannel) downloadFromURL(url, localPath string) error { + resp, err := http.Get(url) + if err != nil { + return fmt.Errorf("failed to download: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download failed with status: %d", resp.StatusCode) + } + + out, err := os.Create(localPath) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + defer out.Close() + + _, err = io.Copy(out, resp.Body) + if err != nil { + return fmt.Errorf("failed to write file: %w", err) + } + + log.Printf("File downloaded successfully to: %s", localPath) + return nil +} + func (c *TelegramChannel) downloadFile(fileID, ext string) string { file, err := c.bot.GetFile(tgbotapi.FileConfig{FileID: fileID}) if err != nil { @@ -331,9 +372,20 @@ func (c *TelegramChannel) downloadFile(fileID, ext string) string { url := file.Link(c.bot.Token) log.Printf("File URL: %s", url) - mediaDir := "/tmp/picoclaw_media" + mediaDir := filepath.Join(os.TempDir(), "picoclaw_media") + if err := os.MkdirAll(mediaDir, 0755); err != nil { + log.Printf("Failed to create media directory: %v", err) + return "" + } - return fmt.Sprintf("%s/%s%s", mediaDir, fileID[:16], ext) + localPath := filepath.Join(mediaDir, fileID[:16]+ext) + + if err := c.downloadFromURL(url, localPath); err != nil { + log.Printf("Failed to download file: %v", err) + return "" + } + + return localPath } func parseChatID(chatIDStr string) (int64, error) { diff --git a/pkg/voice/transcriber.go b/pkg/voice/transcriber.go index 2472ed0..9a09c5e 100644 --- a/pkg/voice/transcriber.go +++ b/pkg/voice/transcriber.go @@ -6,12 +6,13 @@ import ( "encoding/json" "fmt" "io" - "log" "mime/multipart" "net/http" "os" "path/filepath" "time" + + "github.com/sipeed/picoclaw/pkg/logger" ) type GroqTranscriber struct { @@ -27,6 +28,8 @@ type TranscriptionResponse struct { } func NewGroqTranscriber(apiKey string) *GroqTranscriber { + logger.DebugCF("voice", "Creating Groq transcriber", map[string]interface{}{"has_api_key": apiKey != ""}) + apiBase := "https://api.groq.com/openai/v1" return &GroqTranscriber{ apiKey: apiKey, @@ -38,79 +41,125 @@ func NewGroqTranscriber(apiKey string) *GroqTranscriber { } func (t *GroqTranscriber) Transcribe(ctx context.Context, audioFilePath string) (*TranscriptionResponse, error) { - log.Printf("Starting transcription for audio file: %s", audioFilePath) + logger.InfoCF("voice", "Starting transcription", map[string]interface{}{"audio_file": audioFilePath}) audioFile, err := os.Open(audioFilePath) if err != nil { + logger.ErrorCF("voice", "Failed to open audio file", map[string]interface{}{"path": audioFilePath, "error": err}) return nil, fmt.Errorf("failed to open audio file: %w", err) } defer audioFile.Close() fileInfo, err := audioFile.Stat() if err != nil { + logger.ErrorCF("voice", "Failed to get file info", map[string]interface{}{"path": audioFilePath, "error": err}) return nil, fmt.Errorf("failed to get file info: %w", err) } + logger.DebugCF("voice", "Audio file details", map[string]interface{}{ + "size_bytes": fileInfo.Size(), + "file_name": filepath.Base(audioFilePath), + }) + var requestBody bytes.Buffer writer := multipart.NewWriter(&requestBody) part, err := writer.CreateFormFile("file", filepath.Base(audioFilePath)) if err != nil { + logger.ErrorCF("voice", "Failed to create form file", map[string]interface{}{"error": err}) return nil, fmt.Errorf("failed to create form file: %w", err) } - if _, err := io.Copy(part, audioFile); err != nil { + copied, err := io.Copy(part, audioFile) + if err != nil { + logger.ErrorCF("voice", "Failed to copy file content", map[string]interface{}{"error": err}) return nil, fmt.Errorf("failed to copy file content: %w", err) } + logger.DebugCF("voice", "File copied to request", map[string]interface{}{"bytes_copied": copied}) + if err := writer.WriteField("model", "whisper-large-v3"); err != nil { + logger.ErrorCF("voice", "Failed to write model field", map[string]interface{}{"error": err}) return nil, fmt.Errorf("failed to write model field: %w", err) } if err := writer.WriteField("response_format", "json"); err != nil { + logger.ErrorCF("voice", "Failed to write response_format field", map[string]interface{}{"error": err}) return nil, fmt.Errorf("failed to write response_format field: %w", err) } if err := writer.Close(); err != nil { + logger.ErrorCF("voice", "Failed to close multipart writer", map[string]interface{}{"error": err}) return nil, fmt.Errorf("failed to close multipart writer: %w", err) } url := t.apiBase + "/audio/transcriptions" req, err := http.NewRequestWithContext(ctx, "POST", url, &requestBody) if err != nil { + logger.ErrorCF("voice", "Failed to create request", map[string]interface{}{"error": err}) return nil, fmt.Errorf("failed to create request: %w", err) } req.Header.Set("Content-Type", writer.FormDataContentType()) req.Header.Set("Authorization", "Bearer "+t.apiKey) - log.Printf("Sending transcription request to Groq API (file size: %d bytes)", fileInfo.Size()) + logger.DebugCF("voice", "Sending transcription request to Groq API", map[string]interface{}{ + "url": url, + "request_size_bytes": requestBody.Len(), + "file_size_bytes": fileInfo.Size(), + }) resp, err := t.httpClient.Do(req) if err != nil { + logger.ErrorCF("voice", "Failed to send request", map[string]interface{}{"error": err}) return nil, fmt.Errorf("failed to send request: %w", err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { + logger.ErrorCF("voice", "Failed to read response", map[string]interface{}{"error": err}) return nil, fmt.Errorf("failed to read response: %w", err) } if resp.StatusCode != http.StatusOK { + logger.ErrorCF("voice", "API error", map[string]interface{}{ + "status_code": resp.StatusCode, + "response": string(body), + }) return nil, fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(body)) } + logger.DebugCF("voice", "Received response from Groq API", map[string]interface{}{ + "status_code": resp.StatusCode, + "response_size_bytes": len(body), + }) + var result TranscriptionResponse if err := json.Unmarshal(body, &result); err != nil { + logger.ErrorCF("voice", "Failed to unmarshal response", map[string]interface{}{"error": err}) return nil, fmt.Errorf("failed to unmarshal response: %w", err) } - log.Printf("Transcription completed successfully (text length: %d chars)", len(result.Text)) + logger.InfoCF("voice", "Transcription completed successfully", map[string]interface{}{ + "text_length": len(result.Text), + "language": result.Language, + "duration_seconds": result.Duration, + "transcription_preview": truncateText(result.Text, 50), + }) return &result, nil } func (t *GroqTranscriber) IsAvailable() bool { - return t.apiKey != "" + available := t.apiKey != "" + logger.DebugCF("voice", "Checking transcriber availability", map[string]interface{}{"available": available}) + return available +} + +func truncateText(text string, maxLen int) string { + if len(text) <= maxLen { + return text + } + return text[:maxLen] + "..." }