0.29.1原版
This commit is contained in:
anian
@@ -0,0 +1,202 @@
|
||||
// Package gemini implements audiollm.Model against the Gemini generateContent
|
||||
// endpoint. Used by Memos transcription when the user picks a Gemini provider:
|
||||
// the handler issues a transcription instruction via audiollm.Request.Instructions.
|
||||
package gemini
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"mime"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"google.golang.org/genai"
|
||||
|
||||
"github.com/usememos/memos/internal/ai"
|
||||
"github.com/usememos/memos/internal/ai/audio"
|
||||
"github.com/usememos/memos/internal/ai/audiollm"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultEndpoint = "https://generativelanguage.googleapis.com/v1beta"
|
||||
defaultAPIVersion = "v1beta"
|
||||
maxInlineSize = 14 * 1024 * 1024
|
||||
providerName = "Gemini"
|
||||
)
|
||||
|
||||
var supportedContentTypes = map[string]string{
|
||||
"audio/wav": "audio/wav",
|
||||
"audio/x-wav": "audio/wav",
|
||||
"audio/mp3": "audio/mp3",
|
||||
"audio/mpeg": "audio/mp3",
|
||||
"audio/aiff": "audio/aiff",
|
||||
"audio/aac": "audio/aac",
|
||||
"audio/ogg": "audio/ogg",
|
||||
"audio/flac": "audio/flac",
|
||||
"audio/x-flac": "audio/flac",
|
||||
}
|
||||
|
||||
// Model implements audiollm.Model for Gemini generateContent.
|
||||
type Model struct {
|
||||
client *genai.Client
|
||||
}
|
||||
|
||||
// New constructs a Model from a provider config.
|
||||
func New(cfg ai.ProviderConfig, options audiollm.Options) (*Model, error) {
|
||||
endpoint, err := normalizeEndpoint(cfg.Endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg.APIKey == "" {
|
||||
return nil, errors.Errorf("%s API key is required", providerName)
|
||||
}
|
||||
baseURL, apiVersion, err := splitEndpoint(endpoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
httpOptions := genai.HTTPOptions{BaseURL: baseURL, APIVersion: apiVersion}
|
||||
if options.HTTPClient != nil && options.HTTPClient.Timeout > 0 {
|
||||
timeout := options.HTTPClient.Timeout
|
||||
httpOptions.Timeout = &timeout
|
||||
}
|
||||
client, err := genai.NewClient(context.Background(), &genai.ClientConfig{
|
||||
APIKey: cfg.APIKey,
|
||||
Backend: genai.BackendGeminiAPI,
|
||||
HTTPClient: options.HTTPClient,
|
||||
HTTPOptions: httpOptions,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to create Gemini client")
|
||||
}
|
||||
return &Model{client: client}, nil
|
||||
}
|
||||
|
||||
// GenerateFromAudio calls Gemini generateContent with the audio attached.
|
||||
func (m *Model) GenerateFromAudio(ctx context.Context, req audiollm.Request) (*audiollm.Response, error) {
|
||||
if strings.TrimSpace(req.Model) == "" {
|
||||
return nil, errors.New("model is required")
|
||||
}
|
||||
if req.Audio == nil {
|
||||
return nil, errors.New("audio is required")
|
||||
}
|
||||
if strings.TrimSpace(req.Instructions) == "" {
|
||||
return nil, errors.New("instructions are required")
|
||||
}
|
||||
|
||||
audioBytes, err := io.ReadAll(req.Audio)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to read audio")
|
||||
}
|
||||
if len(audioBytes) == 0 {
|
||||
return nil, errors.New("audio is required")
|
||||
}
|
||||
|
||||
contentType := req.ContentType
|
||||
if audio.IsWebMContentType(contentType) {
|
||||
wav, err := audio.WebMOpusToWAV(audioBytes)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to transcode webm audio for Gemini")
|
||||
}
|
||||
audioBytes = wav
|
||||
contentType = "audio/wav"
|
||||
}
|
||||
|
||||
if len(audioBytes) > maxInlineSize {
|
||||
return nil, errors.Errorf("audio is too large for Gemini inline request; maximum size is %d bytes", maxInlineSize)
|
||||
}
|
||||
|
||||
contentType, err = normalizeContentType(contentType)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg := &genai.GenerateContentConfig{}
|
||||
if req.Temperature != nil {
|
||||
t := *req.Temperature
|
||||
cfg.Temperature = &t
|
||||
}
|
||||
|
||||
resp, err := m.client.Models.GenerateContent(ctx, normalizeModelName(req.Model), []*genai.Content{
|
||||
genai.NewContentFromParts([]*genai.Part{
|
||||
genai.NewPartFromBytes(audioBytes, contentType),
|
||||
genai.NewPartFromText(req.Instructions),
|
||||
}, genai.RoleUser),
|
||||
}, cfg)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to send Gemini request")
|
||||
}
|
||||
|
||||
return &audiollm.Response{
|
||||
Text: strings.TrimSpace(resp.Text()),
|
||||
FinishReason: mapFinishReason(resp),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func mapFinishReason(resp *genai.GenerateContentResponse) audiollm.FinishReason {
|
||||
if resp == nil || len(resp.Candidates) == 0 {
|
||||
return audiollm.FinishOther
|
||||
}
|
||||
switch resp.Candidates[0].FinishReason {
|
||||
case genai.FinishReasonStop:
|
||||
return audiollm.FinishStop
|
||||
case genai.FinishReasonMaxTokens:
|
||||
return audiollm.FinishLength
|
||||
case genai.FinishReasonSafety,
|
||||
genai.FinishReasonRecitation,
|
||||
genai.FinishReasonProhibitedContent,
|
||||
genai.FinishReasonSPII,
|
||||
genai.FinishReasonBlocklist,
|
||||
genai.FinishReasonImageSafety,
|
||||
genai.FinishReasonImageProhibitedContent,
|
||||
genai.FinishReasonImageRecitation:
|
||||
return audiollm.FinishSafety
|
||||
default:
|
||||
return audiollm.FinishOther
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeEndpoint(endpoint string) (string, error) {
|
||||
endpoint = strings.TrimSpace(endpoint)
|
||||
if endpoint == "" {
|
||||
endpoint = defaultEndpoint
|
||||
}
|
||||
if _, err := url.ParseRequestURI(endpoint); err != nil {
|
||||
return "", errors.Wrapf(err, "invalid %s endpoint", providerName)
|
||||
}
|
||||
return strings.TrimRight(endpoint, "/"), nil
|
||||
}
|
||||
|
||||
func splitEndpoint(endpoint string) (string, string, error) {
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return "", "", errors.Wrap(err, "invalid Gemini endpoint")
|
||||
}
|
||||
path := strings.TrimRight(parsed.Path, "/")
|
||||
apiVersion := defaultAPIVersion
|
||||
for _, supported := range []string{"v1alpha", "v1beta", "v1"} {
|
||||
if path == "/"+supported || strings.HasSuffix(path, "/"+supported) {
|
||||
apiVersion = supported
|
||||
parsed.Path = strings.TrimSuffix(path, "/"+supported)
|
||||
break
|
||||
}
|
||||
}
|
||||
return strings.TrimRight(parsed.String(), "/"), apiVersion, nil
|
||||
}
|
||||
|
||||
func normalizeContentType(contentType string) (string, error) {
|
||||
mediaType, _, err := mime.ParseMediaType(strings.TrimSpace(contentType))
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "invalid audio content type")
|
||||
}
|
||||
mediaType = strings.ToLower(mediaType)
|
||||
normalized, ok := supportedContentTypes[mediaType]
|
||||
if !ok {
|
||||
return "", errors.Errorf("audio content type %q is not supported by Gemini", mediaType)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
func normalizeModelName(model string) string {
|
||||
return strings.TrimPrefix(strings.TrimSpace(model), "models/")
|
||||
}
|
||||
Reference in New Issue
Block a user