0.29.1原版
This commit is contained in:
anian
@@ -0,0 +1,41 @@
|
||||
// Package audiollm defines the multimodal-audio capability for AI providers.
|
||||
// Implementations call chat-completions or generate-content style APIs that
|
||||
// accept audio as input. For deterministic transcription, prefer internal/ai/stt
|
||||
// where a dedicated STT endpoint exists.
|
||||
package audiollm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Model invokes a multimodal LLM with audio input.
|
||||
type Model interface {
|
||||
GenerateFromAudio(ctx context.Context, req Request) (*Response, error)
|
||||
}
|
||||
|
||||
// Request is the input to a multimodal-audio call.
|
||||
type Request struct {
|
||||
Audio io.Reader
|
||||
Size int64
|
||||
ContentType string
|
||||
Model string
|
||||
Instructions string // literal instruction the model is expected to follow
|
||||
Temperature *float32 // optional; nil leaves the provider default in place
|
||||
}
|
||||
|
||||
// Response is the output of a multimodal-audio call.
|
||||
type Response struct {
|
||||
Text string
|
||||
FinishReason FinishReason
|
||||
}
|
||||
|
||||
// FinishReason describes why the model stopped generating.
|
||||
type FinishReason string
|
||||
|
||||
const (
|
||||
FinishStop FinishReason = "stop" // model finished normally
|
||||
FinishLength FinishReason = "length" // truncated by max-tokens
|
||||
FinishSafety FinishReason = "safety" // safety filter blocked output
|
||||
FinishOther FinishReason = "other" // anything else, including unknown
|
||||
)
|
||||
@@ -0,0 +1,202 @@
|
||||
// Package gemini implements audiollm.Model against the Gemini generateContent
|
||||
// endpoint. Used by Memos transcription when the user picks a Gemini provider:
|
||||
// the handler issues a transcription instruction via audiollm.Request.Instructions.
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"mime"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"google.golang.org/genai"
|
||||
|
||||
"github.com/usememos/memos/internal/ai"
|
||||
"github.com/usememos/memos/internal/ai/audio"
|
||||
"github.com/usememos/memos/internal/ai/audiollm"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultEndpoint = "https://generativelanguage.googleapis.com/v1beta"
|
||||
defaultAPIVersion = "v1beta"
|
||||
maxInlineSize = 14 * 1024 * 1024
|
||||
providerName = "Gemini"
|
||||
)
|
||||
|
||||
var supportedContentTypes = map[string]string{
|
||||
"audio/wav": "audio/wav",
|
||||
"audio/x-wav": "audio/wav",
|
||||
"audio/mp3": "audio/mp3",
|
||||
"audio/mpeg": "audio/mp3",
|
||||
"audio/aiff": "audio/aiff",
|
||||
"audio/aac": "audio/aac",
|
||||
"audio/ogg": "audio/ogg",
|
||||
"audio/flac": "audio/flac",
|
||||
"audio/x-flac": "audio/flac",
|
||||
}
|
||||
|
||||
// Model implements audiollm.Model for Gemini generateContent.
|
||||
type Model struct {
|
||||
client *genai.Client
|
||||
}
|
||||
|
||||
// New constructs a Model from a provider config.
|
||||
func New(cfg ai.ProviderConfig, options audiollm.Options) (*Model, error) {
|
||||
endpoint, err := normalizeEndpoint(cfg.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg.APIKey == "" {
|
||||
return nil, errors.Errorf("%s API key is required", providerName)
|
||||
}
|
||||
baseURL, apiVersion, err := splitEndpoint(endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
httpOptions := genai.HTTPOptions{BaseURL: baseURL, APIVersion: apiVersion}
|
||||
if options.HTTPClient != nil && options.HTTPClient.Timeout > 0 {
|
||||
timeout := options.HTTPClient.Timeout
|
||||
httpOptions.Timeout = &timeout
|
||||
}
|
||||
client, err := genai.NewClient(context.Background(), &genai.ClientConfig{
|
||||
APIKey: cfg.APIKey,
|
||||
Backend: genai.BackendGeminiAPI,
|
||||
HTTPClient: options.HTTPClient,
|
||||
HTTPOptions: httpOptions,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create Gemini client")
|
||||
}
|
||||
return &Model{client: client}, nil
|
||||
}
|
||||
|
||||
// GenerateFromAudio calls Gemini generateContent with the audio attached.
|
||||
func (m *Model) GenerateFromAudio(ctx context.Context, req audiollm.Request) (*audiollm.Response, error) {
|
||||
if strings.TrimSpace(req.Model) == "" {
|
||||
return nil, errors.New("model is required")
|
||||
}
|
||||
if req.Audio == nil {
|
||||
return nil, errors.New("audio is required")
|
||||
}
|
||||
if strings.TrimSpace(req.Instructions) == "" {
|
||||
return nil, errors.New("instructions are required")
|
||||
}
|
||||
|
||||
audioBytes, err := io.ReadAll(req.Audio)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read audio")
|
||||
}
|
||||
if len(audioBytes) == 0 {
|
||||
return nil, errors.New("audio is required")
|
||||
}
|
||||
|
||||
contentType := req.ContentType
|
||||
if audio.IsWebMContentType(contentType) {
|
||||
wav, err := audio.WebMOpusToWAV(audioBytes)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to transcode webm audio for Gemini")
|
||||
}
|
||||
audioBytes = wav
|
||||
contentType = "audio/wav"
|
||||
}
|
||||
|
||||
if len(audioBytes) > maxInlineSize {
|
||||
return nil, errors.Errorf("audio is too large for Gemini inline request; maximum size is %d bytes", maxInlineSize)
|
||||
}
|
||||
|
||||
contentType, err = normalizeContentType(contentType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg := &genai.GenerateContentConfig{}
|
||||
if req.Temperature != nil {
|
||||
t := *req.Temperature
|
||||
cfg.Temperature = &t
|
||||
}
|
||||
|
||||
resp, err := m.client.Models.GenerateContent(ctx, normalizeModelName(req.Model), []*genai.Content{
|
||||
genai.NewContentFromParts([]*genai.Part{
|
||||
genai.NewPartFromBytes(audioBytes, contentType),
|
||||
genai.NewPartFromText(req.Instructions),
|
||||
}, genai.RoleUser),
|
||||
}, cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to send Gemini request")
|
||||
}
|
||||
|
||||
return &audiollm.Response{
|
||||
Text: strings.TrimSpace(resp.Text()),
|
||||
FinishReason: mapFinishReason(resp),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func mapFinishReason(resp *genai.GenerateContentResponse) audiollm.FinishReason {
|
||||
if resp == nil || len(resp.Candidates) == 0 {
|
||||
return audiollm.FinishOther
|
||||
}
|
||||
switch resp.Candidates[0].FinishReason {
|
||||
case genai.FinishReasonStop:
|
||||
return audiollm.FinishStop
|
||||
case genai.FinishReasonMaxTokens:
|
||||
return audiollm.FinishLength
|
||||
case genai.FinishReasonSafety,
|
||||
genai.FinishReasonRecitation,
|
||||
genai.FinishReasonProhibitedContent,
|
||||
genai.FinishReasonSPII,
|
||||
genai.FinishReasonBlocklist,
|
||||
genai.FinishReasonImageSafety,
|
||||
genai.FinishReasonImageProhibitedContent,
|
||||
genai.FinishReasonImageRecitation:
|
||||
return audiollm.FinishSafety
|
||||
default:
|
||||
return audiollm.FinishOther
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeEndpoint(endpoint string) (string, error) {
|
||||
endpoint = strings.TrimSpace(endpoint)
|
||||
if endpoint == "" {
|
||||
endpoint = defaultEndpoint
|
||||
}
|
||||
if _, err := url.ParseRequestURI(endpoint); err != nil {
|
||||
return "", errors.Wrapf(err, "invalid %s endpoint", providerName)
|
||||
}
|
||||
return strings.TrimRight(endpoint, "/"), nil
|
||||
}
|
||||
|
||||
func splitEndpoint(endpoint string) (string, string, error) {
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return "", "", errors.Wrap(err, "invalid Gemini endpoint")
|
||||
}
|
||||
path := strings.TrimRight(parsed.Path, "/")
|
||||
apiVersion := defaultAPIVersion
|
||||
for _, supported := range []string{"v1alpha", "v1beta", "v1"} {
|
||||
if path == "/"+supported || strings.HasSuffix(path, "/"+supported) {
|
||||
apiVersion = supported
|
||||
parsed.Path = strings.TrimSuffix(path, "/"+supported)
|
||||
break
|
||||
}
|
||||
}
|
||||
return strings.TrimRight(parsed.String(), "/"), apiVersion, nil
|
||||
}
|
||||
|
||||
func normalizeContentType(contentType string) (string, error) {
|
||||
mediaType, _, err := mime.ParseMediaType(strings.TrimSpace(contentType))
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "invalid audio content type")
|
||||
}
|
||||
mediaType = strings.ToLower(mediaType)
|
||||
normalized, ok := supportedContentTypes[mediaType]
|
||||
if !ok {
|
||||
return "", errors.Errorf("audio content type %q is not supported by Gemini", mediaType)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func normalizeModelName(model string) string {
|
||||
return strings.TrimPrefix(strings.TrimSpace(model), "models/")
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package gemini_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/usememos/memos/internal/ai"
|
||||
"github.com/usememos/memos/internal/ai/audiollm"
|
||||
audiollmgemini "github.com/usememos/memos/internal/ai/audiollm/gemini"
|
||||
)
|
||||
|
||||
func TestGenerateFromAudio(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
require.Equal(t, http.MethodPost, r.Method)
|
||||
require.Equal(t, "/v1beta/models/gemini-2.5-flash:generateContent", r.URL.Path)
|
||||
require.Equal(t, "test-key", r.Header.Get("x-goog-api-key"))
|
||||
require.Equal(t, "application/json", r.Header.Get("Content-Type"))
|
||||
|
||||
var request struct {
|
||||
Contents []struct {
|
||||
Parts []struct {
|
||||
Text string `json:"text"`
|
||||
InlineData *struct {
|
||||
MIMEType string `json:"mimeType"`
|
||||
Data string `json:"data"`
|
||||
} `json:"inlineData"`
|
||||
} `json:"parts"`
|
||||
} `json:"contents"`
|
||||
GenerationConfig map[string]json.Number `json:"generationConfig"`
|
||||
}
|
||||
require.NoError(t, json.NewDecoder(r.Body).Decode(&request))
|
||||
require.Len(t, request.Contents, 1)
|
||||
require.Len(t, request.Contents[0].Parts, 2)
|
||||
require.NotNil(t, request.Contents[0].Parts[0].InlineData)
|
||||
require.Equal(t, "audio/mp3", request.Contents[0].Parts[0].InlineData.MIMEType)
|
||||
audio, err := base64.StdEncoding.DecodeString(request.Contents[0].Parts[0].InlineData.Data)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "audio bytes", string(audio))
|
||||
require.Equal(t, "transcribe please", request.Contents[0].Parts[1].Text)
|
||||
require.Equal(t, json.Number("0"), request.GenerationConfig["temperature"])
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
|
||||
"candidates": []map[string]any{
|
||||
{
|
||||
"finishReason": "STOP",
|
||||
"content": map[string]any{
|
||||
"parts": []map[string]string{{"text": "hello from gemini"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
model, err := audiollmgemini.New(ai.ProviderConfig{
|
||||
Type: ai.ProviderGemini,
|
||||
Endpoint: server.URL + "/v1beta",
|
||||
APIKey: "test-key",
|
||||
}, audiollm.ApplyOptions(nil))
|
||||
require.NoError(t, err)
|
||||
|
||||
temp := float32(0)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
resp, err := model.GenerateFromAudio(ctx, audiollm.Request{
|
||||
Model: "models/gemini-2.5-flash",
|
||||
ContentType: "audio/mpeg",
|
||||
Audio: strings.NewReader("audio bytes"),
|
||||
Instructions: "transcribe please",
|
||||
Temperature: &temp,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "hello from gemini", resp.Text)
|
||||
require.Equal(t, audiollm.FinishStop, resp.FinishReason)
|
||||
}
|
||||
|
||||
func TestGenerateFromAudioRejectsUnsupportedContentType(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
model, err := audiollmgemini.New(ai.ProviderConfig{
|
||||
Type: ai.ProviderGemini,
|
||||
Endpoint: "https://example.com/v1beta",
|
||||
APIKey: "test-key",
|
||||
}, audiollm.ApplyOptions(nil))
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = model.GenerateFromAudio(context.Background(), audiollm.Request{
|
||||
Model: "gemini-2.5-flash",
|
||||
ContentType: "video/mp4",
|
||||
Audio: strings.NewReader("video bytes"),
|
||||
Instructions: "transcribe please",
|
||||
})
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "not supported by Gemini")
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package audiollm
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
const defaultHTTPTimeout = 2 * time.Minute
|
||||
|
||||
// Options is the resolved option set passed to provider implementations.
|
||||
type Options struct {
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// ModelOption customizes a Model.
|
||||
type ModelOption func(*Options)
|
||||
|
||||
// WithHTTPClient overrides the HTTP client used by the model.
|
||||
func WithHTTPClient(client *http.Client) ModelOption {
|
||||
return func(o *Options) {
|
||||
if client != nil {
|
||||
o.HTTPClient = client
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ApplyOptions resolves a ModelOption slice into Options with defaults.
|
||||
func ApplyOptions(opts []ModelOption) Options {
|
||||
resolved := Options{HTTPClient: &http.Client{Timeout: defaultHTTPTimeout}}
|
||||
for _, apply := range opts {
|
||||
apply(&resolved)
|
||||
}
|
||||
return resolved
|
||||
}
|
||||
Reference in New Issue
Block a user