0.29.1原版

internal/ai/ai.go

@@ -0,0 +1,20 @@
+package ai
+
+// ProviderType identifies an AI provider implementation.
+type ProviderType string
+
+const (
+	// ProviderOpenAI is OpenAI's hosted API.
+	ProviderOpenAI ProviderType = "OPENAI"
+	// ProviderGemini is Google's Gemini API.
+	ProviderGemini ProviderType = "GEMINI"
+)
+
+// ProviderConfig configures a callable AI provider connection.
+type ProviderConfig struct {
+	ID       string
+	Title    string
+	Type     ProviderType
+	Endpoint string
+	APIKey   string
+}

internal/ai/audio/webm.go

@@ -0,0 +1,161 @@
+// Package audio provides audio container/codec helpers for AI providers.
+//
+// The motivating use case is Gemini transcription: Gemini's audio inputs
+// require WAV/MP3/AIFF/AAC/OGG/FLAC, but browser MediaRecorder defaults to
+// WebM/Opus. This package converts WebM/Opus into 16-bit PCM WAV using
+// pure-Go decoders — no ffmpeg or other system dependency.
+package audio
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+	"strings"
+
+	"github.com/at-wat/ebml-go"
+	"github.com/at-wat/ebml-go/webm"
+	"github.com/pion/opus"
+	"github.com/pkg/errors"
+)
+
+const (
+	opusOutputSampleRate = 48000
+	// maxOpusPacketSamples is Opus's spec maximum: 120 ms at 48 kHz.
+	maxOpusPacketSamples = 5760
+	// opusCodecID is the WebM TrackEntry CodecID for an Opus audio track.
+	opusCodecID = "A_OPUS"
+	// opusHeadMinLength is the minimum size of the OpusHead identification
+	// header stored in TrackEntry.CodecPrivate.
+	opusHeadMinLength = 19
+)
+
+// WebMOpusToWAV decodes a WebM/Opus file into 16-bit PCM WAV bytes.
+//
+// The output is mono or stereo at 48 kHz (Opus's native decode rate),
+// regardless of the original encoder's hint. Pre-skip samples declared in
+// the OpusHead are discarded to avoid the encoder's startup padding.
+//
+// The function reads the entire WebM document into memory; callers should
+// enforce their own size limits before invoking it.
+func WebMOpusToWAV(input []byte) ([]byte, error) {
+	var doc struct {
+		Header  webm.EBMLHeader `ebml:"EBML"`
+		Segment webm.Segment    `ebml:"Segment"`
+	}
+	if err := ebml.Unmarshal(bytes.NewReader(input), &doc); err != nil && !errors.Is(err, io.EOF) {
+		return nil, errors.Wrap(err, "parse webm")
+	}
+
+	track := findOpusTrack(doc.Segment.Tracks.TrackEntry)
+	if track == nil {
+		return nil, errors.New("webm has no Opus audio track")
+	}
+	if len(track.CodecPrivate) < opusHeadMinLength {
+		return nil, errors.Errorf("invalid OpusHead: expected at least %d bytes, got %d", opusHeadMinLength, len(track.CodecPrivate))
+	}
+
+	channels := int(track.Audio.Channels)
+	if channels < 1 || channels > 2 {
+		return nil, errors.Errorf("unsupported Opus channel count: %d", channels)
+	}
+	preSkip := int(binary.LittleEndian.Uint16(track.CodecPrivate[10:12]))
+
+	decoder := opus.NewDecoder()
+	if err := decoder.Init(opusOutputSampleRate, channels); err != nil {
+		return nil, errors.Wrap(err, "init opus decoder")
+	}
+
+	pcm := make([]int16, 0, 1<<16)
+	frame := make([]int16, maxOpusPacketSamples*channels)
+
+	decodeBlock := func(block ebml.Block) error {
+		if block.TrackNumber != track.TrackNumber {
+			return nil
+		}
+		for _, packet := range block.Data {
+			if len(packet) == 0 {
+				continue
+			}
+			n, err := decoder.DecodeToInt16(packet, frame)
+			if err != nil {
+				return errors.Wrap(err, "decode opus packet")
+			}
+			pcm = append(pcm, frame[:n*channels]...)
+		}
+		return nil
+	}
+
+	for _, cluster := range doc.Segment.Cluster {
+		for _, sb := range cluster.SimpleBlock {
+			if err := decodeBlock(sb); err != nil {
+				return nil, err
+			}
+		}
+		for _, bg := range cluster.BlockGroup {
+			if err := decodeBlock(bg.Block); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	skip := preSkip * channels
+	if skip > len(pcm) {
+		skip = len(pcm)
+	}
+	pcm = pcm[skip:]
+
+	return encodeWAV(pcm, opusOutputSampleRate, channels), nil
+}
+
+// IsWebMContentType reports whether the MIME type is WebM audio.
+// Both "audio/webm" and "audio/webm; codecs=opus" return true.
+func IsWebMContentType(contentType string) bool {
+	contentType = strings.TrimSpace(contentType)
+	if contentType == "" {
+		return false
+	}
+	if i := strings.IndexByte(contentType, ';'); i >= 0 {
+		contentType = contentType[:i]
+	}
+	return strings.EqualFold(strings.TrimSpace(contentType), "audio/webm")
+}
+
+func findOpusTrack(entries []webm.TrackEntry) *webm.TrackEntry {
+	for i := range entries {
+		entry := &entries[i]
+		if entry.CodecID == opusCodecID && entry.Audio != nil {
+			return entry
+		}
+	}
+	return nil
+}
+
+// encodeWAV writes a standard RIFF/WAVE container around 16-bit PCM samples.
+// Reference layout: http://soundfile.sapp.org/doc/WaveFormat/
+func encodeWAV(samples []int16, sampleRate, channels int) []byte {
+	const bitsPerSample = 16
+	const bytesPerSample = bitsPerSample / 8
+	blockAlign := channels * bytesPerSample
+	byteRate := sampleRate * blockAlign
+	dataSize := len(samples) * bytesPerSample
+
+	buf := bytes.NewBuffer(make([]byte, 0, 44+dataSize))
+	buf.WriteString("RIFF")
+	_ = binary.Write(buf, binary.LittleEndian, uint32(36+dataSize))
+	buf.WriteString("WAVE")
+
+	buf.WriteString("fmt ")
+	_ = binary.Write(buf, binary.LittleEndian, uint32(16))
+	_ = binary.Write(buf, binary.LittleEndian, uint16(1)) // PCM
+	_ = binary.Write(buf, binary.LittleEndian, uint16(channels))
+	_ = binary.Write(buf, binary.LittleEndian, uint32(sampleRate))
+	_ = binary.Write(buf, binary.LittleEndian, uint32(byteRate))
+	_ = binary.Write(buf, binary.LittleEndian, uint16(blockAlign))
+	_ = binary.Write(buf, binary.LittleEndian, uint16(bitsPerSample))
+
+	buf.WriteString("data")
+	_ = binary.Write(buf, binary.LittleEndian, uint32(dataSize))
+	_ = binary.Write(buf, binary.LittleEndian, samples)
+
+	return buf.Bytes()
+}

internal/ai/audio/webm_test.go

@@ -0,0 +1,48 @@
+package audio
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestIsWebMContentType(t *testing.T) {
+	cases := []struct {
+		in   string
+		want bool
+	}{
+		{"audio/webm", true},
+		{"audio/webm;codecs=opus", true},
+		{"audio/webm; codecs=opus", true},
+		{"AUDIO/WEBM", true},
+		{"  audio/webm  ", true},
+		{"audio/wav", false},
+		{"audio/mp4", false},
+		{"video/webm", false},
+		{"", false},
+		{"webm", false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.in, func(t *testing.T) {
+			require.Equal(t, tc.want, IsWebMContentType(tc.in))
+		})
+	}
+}
+
+func TestWebMOpusToWAV_RejectsInvalidInput(t *testing.T) {
+	t.Run("empty", func(t *testing.T) {
+		_, err := WebMOpusToWAV(nil)
+		require.Error(t, err)
+	})
+
+	t.Run("not webm", func(t *testing.T) {
+		_, err := WebMOpusToWAV([]byte("hello world this is not webm"))
+		require.Error(t, err)
+	})
+
+	t.Run("truncated webm header bytes", func(t *testing.T) {
+		// Valid EBML magic but no Segment.
+		_, err := WebMOpusToWAV([]byte{0x1A, 0x45, 0xDF, 0xA3})
+		require.Error(t, err)
+	})
+}

internal/ai/audiollm/audiollm.go

@@ -0,0 +1,41 @@
+// Package audiollm defines the multimodal-audio capability for AI providers.
+// Implementations call chat-completions or generate-content style APIs that
+// accept audio as input. For deterministic transcription, prefer internal/ai/stt
+// where a dedicated STT endpoint exists.
+package audiollm
+
+import (
+	"context"
+	"io"
+)
+
+// Model invokes a multimodal LLM with audio input.
+type Model interface {
+	GenerateFromAudio(ctx context.Context, req Request) (*Response, error)
+}
+
+// Request is the input to a multimodal-audio call.
+type Request struct {
+	Audio        io.Reader
+	Size         int64
+	ContentType  string
+	Model        string
+	Instructions string   // literal instruction the model is expected to follow
+	Temperature  *float32 // optional; nil leaves the provider default in place
+}
+
+// Response is the output of a multimodal-audio call.
+type Response struct {
+	Text         string
+	FinishReason FinishReason
+}
+
+// FinishReason describes why the model stopped generating.
+type FinishReason string
+
+const (
+	FinishStop   FinishReason = "stop"   // model finished normally
+	FinishLength FinishReason = "length" // truncated by max-tokens
+	FinishSafety FinishReason = "safety" // safety filter blocked output
+	FinishOther  FinishReason = "other"  // anything else, including unknown
+)

internal/ai/audiollm/gemini/gemini.go

@@ -0,0 +1,202 @@
+// Package gemini implements audiollm.Model against the Gemini generateContent
+// endpoint. Used by Memos transcription when the user picks a Gemini provider:
+// the handler issues a transcription instruction via audiollm.Request.Instructions.
+package gemini
+
+import (
+	"context"
+	"io"
+	"mime"
+	"net/url"
+	"strings"
+
+	"github.com/pkg/errors"
+	"google.golang.org/genai"
+
+	"github.com/usememos/memos/internal/ai"
+	"github.com/usememos/memos/internal/ai/audio"
+	"github.com/usememos/memos/internal/ai/audiollm"
+)
+
+const (
+	defaultEndpoint   = "https://generativelanguage.googleapis.com/v1beta"
+	defaultAPIVersion = "v1beta"
+	maxInlineSize     = 14 * 1024 * 1024
+	providerName      = "Gemini"
+)
+
+var supportedContentTypes = map[string]string{
+	"audio/wav":    "audio/wav",
+	"audio/x-wav":  "audio/wav",
+	"audio/mp3":    "audio/mp3",
+	"audio/mpeg":   "audio/mp3",
+	"audio/aiff":   "audio/aiff",
+	"audio/aac":    "audio/aac",
+	"audio/ogg":    "audio/ogg",
+	"audio/flac":   "audio/flac",
+	"audio/x-flac": "audio/flac",
+}
+
+// Model implements audiollm.Model for Gemini generateContent.
+type Model struct {
+	client *genai.Client
+}
+
+// New constructs a Model from a provider config.
+func New(cfg ai.ProviderConfig, options audiollm.Options) (*Model, error) {
+	endpoint, err := normalizeEndpoint(cfg.Endpoint)
+	if err != nil {
+		return nil, err
+	}
+	if cfg.APIKey == "" {
+		return nil, errors.Errorf("%s API key is required", providerName)
+	}
+	baseURL, apiVersion, err := splitEndpoint(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	httpOptions := genai.HTTPOptions{BaseURL: baseURL, APIVersion: apiVersion}
+	if options.HTTPClient != nil && options.HTTPClient.Timeout > 0 {
+		timeout := options.HTTPClient.Timeout
+		httpOptions.Timeout = &timeout
+	}
+	client, err := genai.NewClient(context.Background(), &genai.ClientConfig{
+		APIKey:      cfg.APIKey,
+		Backend:     genai.BackendGeminiAPI,
+		HTTPClient:  options.HTTPClient,
+		HTTPOptions: httpOptions,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create Gemini client")
+	}
+	return &Model{client: client}, nil
+}
+
+// GenerateFromAudio calls Gemini generateContent with the audio attached.
+func (m *Model) GenerateFromAudio(ctx context.Context, req audiollm.Request) (*audiollm.Response, error) {
+	if strings.TrimSpace(req.Model) == "" {
+		return nil, errors.New("model is required")
+	}
+	if req.Audio == nil {
+		return nil, errors.New("audio is required")
+	}
+	if strings.TrimSpace(req.Instructions) == "" {
+		return nil, errors.New("instructions are required")
+	}
+
+	audioBytes, err := io.ReadAll(req.Audio)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read audio")
+	}
+	if len(audioBytes) == 0 {
+		return nil, errors.New("audio is required")
+	}
+
+	contentType := req.ContentType
+	if audio.IsWebMContentType(contentType) {
+		wav, err := audio.WebMOpusToWAV(audioBytes)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to transcode webm audio for Gemini")
+		}
+		audioBytes = wav
+		contentType = "audio/wav"
+	}
+
+	if len(audioBytes) > maxInlineSize {
+		return nil, errors.Errorf("audio is too large for Gemini inline request; maximum size is %d bytes", maxInlineSize)
+	}
+
+	contentType, err = normalizeContentType(contentType)
+	if err != nil {
+		return nil, err
+	}
+
+	cfg := &genai.GenerateContentConfig{}
+	if req.Temperature != nil {
+		t := *req.Temperature
+		cfg.Temperature = &t
+	}
+
+	resp, err := m.client.Models.GenerateContent(ctx, normalizeModelName(req.Model), []*genai.Content{
+		genai.NewContentFromParts([]*genai.Part{
+			genai.NewPartFromBytes(audioBytes, contentType),
+			genai.NewPartFromText(req.Instructions),
+		}, genai.RoleUser),
+	}, cfg)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to send Gemini request")
+	}
+
+	return &audiollm.Response{
+		Text:         strings.TrimSpace(resp.Text()),
+		FinishReason: mapFinishReason(resp),
+	}, nil
+}
+
+func mapFinishReason(resp *genai.GenerateContentResponse) audiollm.FinishReason {
+	if resp == nil || len(resp.Candidates) == 0 {
+		return audiollm.FinishOther
+	}
+	switch resp.Candidates[0].FinishReason {
+	case genai.FinishReasonStop:
+		return audiollm.FinishStop
+	case genai.FinishReasonMaxTokens:
+		return audiollm.FinishLength
+	case genai.FinishReasonSafety,
+		genai.FinishReasonRecitation,
+		genai.FinishReasonProhibitedContent,
+		genai.FinishReasonSPII,
+		genai.FinishReasonBlocklist,
+		genai.FinishReasonImageSafety,
+		genai.FinishReasonImageProhibitedContent,
+		genai.FinishReasonImageRecitation:
+		return audiollm.FinishSafety
+	default:
+		return audiollm.FinishOther
+	}
+}
+
+func normalizeEndpoint(endpoint string) (string, error) {
+	endpoint = strings.TrimSpace(endpoint)
+	if endpoint == "" {
+		endpoint = defaultEndpoint
+	}
+	if _, err := url.ParseRequestURI(endpoint); err != nil {
+		return "", errors.Wrapf(err, "invalid %s endpoint", providerName)
+	}
+	return strings.TrimRight(endpoint, "/"), nil
+}
+
+func splitEndpoint(endpoint string) (string, string, error) {
+	parsed, err := url.Parse(endpoint)
+	if err != nil {
+		return "", "", errors.Wrap(err, "invalid Gemini endpoint")
+	}
+	path := strings.TrimRight(parsed.Path, "/")
+	apiVersion := defaultAPIVersion
+	for _, supported := range []string{"v1alpha", "v1beta", "v1"} {
+		if path == "/"+supported || strings.HasSuffix(path, "/"+supported) {
+			apiVersion = supported
+			parsed.Path = strings.TrimSuffix(path, "/"+supported)
+			break
+		}
+	}
+	return strings.TrimRight(parsed.String(), "/"), apiVersion, nil
+}
+
+func normalizeContentType(contentType string) (string, error) {
+	mediaType, _, err := mime.ParseMediaType(strings.TrimSpace(contentType))
+	if err != nil {
+		return "", errors.Wrap(err, "invalid audio content type")
+	}
+	mediaType = strings.ToLower(mediaType)
+	normalized, ok := supportedContentTypes[mediaType]
+	if !ok {
+		return "", errors.Errorf("audio content type %q is not supported by Gemini", mediaType)
+	}
+	return normalized, nil
+}
+
+func normalizeModelName(model string) string {
+	return strings.TrimPrefix(strings.TrimSpace(model), "models/")
+}

internal/ai/audiollm/gemini/gemini_test.go

@@ -0,0 +1,106 @@
+package gemini_test
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/usememos/memos/internal/ai"
+	"github.com/usememos/memos/internal/ai/audiollm"
+	audiollmgemini "github.com/usememos/memos/internal/ai/audiollm/gemini"
+)
+
+func TestGenerateFromAudio(t *testing.T) {
+	t.Parallel()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, http.MethodPost, r.Method)
+		require.Equal(t, "/v1beta/models/gemini-2.5-flash:generateContent", r.URL.Path)
+		require.Equal(t, "test-key", r.Header.Get("x-goog-api-key"))
+		require.Equal(t, "application/json", r.Header.Get("Content-Type"))
+
+		var request struct {
+			Contents []struct {
+				Parts []struct {
+					Text       string `json:"text"`
+					InlineData *struct {
+						MIMEType string `json:"mimeType"`
+						Data     string `json:"data"`
+					} `json:"inlineData"`
+				} `json:"parts"`
+			} `json:"contents"`
+			GenerationConfig map[string]json.Number `json:"generationConfig"`
+		}
+		require.NoError(t, json.NewDecoder(r.Body).Decode(&request))
+		require.Len(t, request.Contents, 1)
+		require.Len(t, request.Contents[0].Parts, 2)
+		require.NotNil(t, request.Contents[0].Parts[0].InlineData)
+		require.Equal(t, "audio/mp3", request.Contents[0].Parts[0].InlineData.MIMEType)
+		audio, err := base64.StdEncoding.DecodeString(request.Contents[0].Parts[0].InlineData.Data)
+		require.NoError(t, err)
+		require.Equal(t, "audio bytes", string(audio))
+		require.Equal(t, "transcribe please", request.Contents[0].Parts[1].Text)
+		require.Equal(t, json.Number("0"), request.GenerationConfig["temperature"])
+
+		w.Header().Set("Content-Type", "application/json")
+		require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
+			"candidates": []map[string]any{
+				{
+					"finishReason": "STOP",
+					"content": map[string]any{
+						"parts": []map[string]string{{"text": "hello from gemini"}},
+					},
+				},
+			},
+		}))
+	}))
+	defer server.Close()
+
+	model, err := audiollmgemini.New(ai.ProviderConfig{
+		Type:     ai.ProviderGemini,
+		Endpoint: server.URL + "/v1beta",
+		APIKey:   "test-key",
+	}, audiollm.ApplyOptions(nil))
+	require.NoError(t, err)
+
+	temp := float32(0)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	resp, err := model.GenerateFromAudio(ctx, audiollm.Request{
+		Model:        "models/gemini-2.5-flash",
+		ContentType:  "audio/mpeg",
+		Audio:        strings.NewReader("audio bytes"),
+		Instructions: "transcribe please",
+		Temperature:  &temp,
+	})
+	require.NoError(t, err)
+	require.Equal(t, "hello from gemini", resp.Text)
+	require.Equal(t, audiollm.FinishStop, resp.FinishReason)
+}
+
+func TestGenerateFromAudioRejectsUnsupportedContentType(t *testing.T) {
+	t.Parallel()
+
+	model, err := audiollmgemini.New(ai.ProviderConfig{
+		Type:     ai.ProviderGemini,
+		Endpoint: "https://example.com/v1beta",
+		APIKey:   "test-key",
+	}, audiollm.ApplyOptions(nil))
+	require.NoError(t, err)
+
+	_, err = model.GenerateFromAudio(context.Background(), audiollm.Request{
+		Model:        "gemini-2.5-flash",
+		ContentType:  "video/mp4",
+		Audio:        strings.NewReader("video bytes"),
+		Instructions: "transcribe please",
+	})
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "not supported by Gemini")
+}

internal/ai/audiollm/options.go

@@ -0,0 +1,34 @@
+package audiollm
+
+import (
+	"net/http"
+	"time"
+)
+
+const defaultHTTPTimeout = 2 * time.Minute
+
+// Options is the resolved option set passed to provider implementations.
+type Options struct {
+	HTTPClient *http.Client
+}
+
+// ModelOption customizes a Model.
+type ModelOption func(*Options)
+
+// WithHTTPClient overrides the HTTP client used by the model.
+func WithHTTPClient(client *http.Client) ModelOption {
+	return func(o *Options) {
+		if client != nil {
+			o.HTTPClient = client
+		}
+	}
+}
+
+// ApplyOptions resolves a ModelOption slice into Options with defaults.
+func ApplyOptions(opts []ModelOption) Options {
+	resolved := Options{HTTPClient: &http.Client{Timeout: defaultHTTPTimeout}}
+	for _, apply := range opts {
+		apply(&resolved)
+	}
+	return resolved
+}

internal/ai/errors.go

@@ -0,0 +1,17 @@
+package ai
+
+import "github.com/pkg/errors"
+
+var (
+	// ErrProviderNotFound indicates that a requested provider ID does not exist.
+	ErrProviderNotFound = errors.New("AI provider not found")
+	// ErrCapabilityUnsupported indicates that the provider does not support the requested capability.
+	ErrCapabilityUnsupported = errors.New("AI provider capability unsupported")
+	// ErrSTTNotSupported indicates that the provider does not have a dedicated
+	// speech-to-text endpoint. Use the audiollm package for multimodal audio
+	// understanding when this is returned.
+	ErrSTTNotSupported = errors.New("provider does not support speech-to-text capability")
+	// ErrAudioLLMNotSupported indicates that the provider does not have a
+	// multimodal-audio LLM available in this codebase.
+	ErrAudioLLMNotSupported = errors.New("provider does not support multimodal audio capability")
+)

internal/ai/models.go

@@ -0,0 +1,22 @@
+package ai
+
+import "github.com/pkg/errors"
+
+const (
+	// DefaultOpenAITranscriptionModel is the built-in OpenAI transcription model.
+	DefaultOpenAITranscriptionModel = "whisper-1"
+	// DefaultGeminiTranscriptionModel is the built-in Gemini transcription model.
+	DefaultGeminiTranscriptionModel = "gemini-2.5-flash"
+)
+
+// DefaultTranscriptionModel returns the built-in transcription model for a provider.
+func DefaultTranscriptionModel(providerType ProviderType) (string, error) {
+	switch providerType {
+	case ProviderOpenAI:
+		return DefaultOpenAITranscriptionModel, nil
+	case ProviderGemini:
+		return DefaultGeminiTranscriptionModel, nil
+	default:
+		return "", errors.Wrapf(ErrCapabilityUnsupported, "provider type %q", providerType)
+	}
+}

internal/ai/resolver.go

@@ -0,0 +1,16 @@
+package ai
+
+import "github.com/pkg/errors"
+
+// FindProvider returns the provider with the given ID.
+func FindProvider(providers []ProviderConfig, providerID string) (*ProviderConfig, error) {
+	if providerID == "" {
+		return nil, errors.Wrap(ErrProviderNotFound, "provider ID is required")
+	}
+	for _, provider := range providers {
+		if provider.ID == providerID {
+			return &provider, nil
+		}
+	}
+	return nil, errors.Wrapf(ErrProviderNotFound, "provider ID %q", providerID)
+}

internal/ai/stt/openai/openai.go

@@ -0,0 +1,116 @@
+// Package openai implements stt.Transcriber against the OpenAI
+// /audio/transcriptions endpoint (and any compatible third-party endpoint
+// such as Groq Whisper, faster-whisper self-hosted, or Azure Whisper).
+package openai
+
+import (
+	"context"
+	"mime"
+	"net/url"
+	"strings"
+
+	openaisdk "github.com/openai/openai-go/v3"
+	openaioption "github.com/openai/openai-go/v3/option"
+	"github.com/pkg/errors"
+
+	"github.com/usememos/memos/internal/ai"
+	"github.com/usememos/memos/internal/ai/stt"
+)
+
+const defaultEndpoint = "https://api.openai.com/v1"
+
+// Transcriber implements stt.Transcriber for OpenAI-compatible STT endpoints.
+type Transcriber struct {
+	client openaisdk.Client
+}
+
+// New constructs a Transcriber from a provider config.
+func New(cfg ai.ProviderConfig, options stt.Options) (*Transcriber, error) {
+	endpoint, err := normalizeEndpoint(cfg.Endpoint)
+	if err != nil {
+		return nil, err
+	}
+	if cfg.APIKey == "" {
+		return nil, errors.New("OpenAI API key is required")
+	}
+	return &Transcriber{
+		client: openaisdk.NewClient(
+			openaioption.WithAPIKey(cfg.APIKey),
+			openaioption.WithBaseURL(endpoint),
+			openaioption.WithHTTPClient(options.HTTPClient),
+		),
+	}, nil
+}
+
+// Transcribe sends the audio to /audio/transcriptions.
+func (t *Transcriber) Transcribe(ctx context.Context, req stt.Request) (*stt.Response, error) {
+	if strings.TrimSpace(req.Model) == "" {
+		return nil, errors.New("model is required")
+	}
+	if req.Audio == nil {
+		return nil, errors.New("audio is required")
+	}
+
+	filename, contentType, err := normalizeAudioMetadata(req)
+	if err != nil {
+		return nil, err
+	}
+
+	params := openaisdk.AudioTranscriptionNewParams{
+		File:           openaisdk.File(req.Audio, filename, contentType),
+		Model:          openaisdk.AudioModel(req.Model),
+		ResponseFormat: openaisdk.AudioResponseFormatJSON,
+	}
+	if req.Prompt != "" {
+		params.Prompt = openaisdk.String(req.Prompt)
+	}
+	if req.Language != "" {
+		params.Language = openaisdk.String(req.Language)
+	}
+
+	resp, err := t.client.Audio.Transcriptions.New(ctx, params)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to send OpenAI transcription request")
+	}
+	return &stt.Response{
+		Text:     resp.Text,
+		Language: resp.Language,
+	}, nil
+}
+
+func normalizeEndpoint(endpoint string) (string, error) {
+	endpoint = strings.TrimSpace(endpoint)
+	if endpoint == "" {
+		endpoint = defaultEndpoint
+	}
+	if _, err := url.ParseRequestURI(endpoint); err != nil {
+		return "", errors.Wrap(err, "invalid OpenAI endpoint")
+	}
+	return strings.TrimRight(endpoint, "/"), nil
+}
+
+func normalizeAudioMetadata(req stt.Request) (string, string, error) {
+	filename := strings.TrimSpace(req.Filename)
+	if filename == "" {
+		filename = "audio"
+	}
+	contentType := strings.TrimSpace(req.ContentType)
+	if contentType == "" {
+		contentType = "application/octet-stream"
+	} else {
+		mediaType, _, err := mime.ParseMediaType(contentType)
+		if err != nil {
+			return "", "", errors.Wrap(err, "invalid audio content type")
+		}
+		contentType = mediaType
+	}
+	return sanitizeFilename(filename), contentType, nil
+}
+
+func sanitizeFilename(filename string) string {
+	filename = strings.NewReplacer("\r", "_", "\n", "_").Replace(filename)
+	if strings.TrimSpace(filename) == "" {
+		return "audio"
+	}
+	return filename
+}

internal/ai/stt/openai/openai_test.go

@@ -0,0 +1,68 @@
+package openai_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/usememos/memos/internal/ai"
+	"github.com/usememos/memos/internal/ai/stt"
+	sttopenai "github.com/usememos/memos/internal/ai/stt/openai"
+)
+
+func TestTranscribe(t *testing.T) {
+	t.Parallel()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, http.MethodPost, r.Method)
+		require.Equal(t, "/audio/transcriptions", r.URL.Path)
+		require.Equal(t, "Bearer test-key", r.Header.Get("Authorization"))
+		require.NoError(t, r.ParseMultipartForm(10<<20))
+		require.Equal(t, "gpt-4o-transcribe", r.FormValue("model"))
+		require.Equal(t, "json", r.FormValue("response_format"))
+		require.Equal(t, "domain words", r.FormValue("prompt"))
+		require.Equal(t, "en", r.FormValue("language"))
+
+		file, header, err := r.FormFile("file")
+		require.NoError(t, err)
+		defer file.Close()
+		require.Equal(t, "voice.wav", header.Filename)
+		require.Equal(t, "audio/wav", header.Header.Get("Content-Type"))
+
+		w.Header().Set("Content-Type", "application/json")
+		require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
+			"text":     "hello world",
+			"language": "en",
+			"duration": 1.5,
+		}))
+	}))
+	defer server.Close()
+
+	transcriber, err := sttopenai.New(ai.ProviderConfig{
+		Type:     ai.ProviderOpenAI,
+		Endpoint: server.URL,
+		APIKey:   "test-key",
+	}, stt.ApplyOptions(nil))
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	response, err := transcriber.Transcribe(ctx, stt.Request{
+		Model:       "gpt-4o-transcribe",
+		Filename:    "voice.wav",
+		ContentType: "audio/wav",
+		Audio:       strings.NewReader("RIFF"),
+		Prompt:      "domain words",
+		Language:    "en",
+	})
+	require.NoError(t, err)
+	require.Equal(t, "hello world", response.Text)
+	require.Equal(t, "en", response.Language)
+	// Note: Duration intentionally omitted from stt.Response — not exposed in the new contract.
+}

internal/ai/stt/options.go

@@ -0,0 +1,34 @@
+package stt
+
+import (
+	"net/http"
+	"time"
+)
+
+const defaultHTTPTimeout = 2 * time.Minute
+
+// Options is the resolved option set passed to provider implementations.
+type Options struct {
+	HTTPClient *http.Client
+}
+
+// TranscriberOption customizes a Transcriber.
+type TranscriberOption func(*Options)
+
+// WithHTTPClient overrides the HTTP client used by the transcriber.
+func WithHTTPClient(client *http.Client) TranscriberOption {
+	return func(o *Options) {
+		if client != nil {
+			o.HTTPClient = client
+		}
+	}
+}
+
+// ApplyOptions resolves a TranscriberOption slice into Options with defaults.
+func ApplyOptions(opts []TranscriberOption) Options {
+	resolved := Options{HTTPClient: &http.Client{Timeout: defaultHTTPTimeout}}
+	for _, apply := range opts {
+		apply(&resolved)
+	}
+	return resolved
+}

internal/ai/stt/stt.go

@@ -0,0 +1,41 @@
+// Package stt defines the speech-to-text capability for AI providers.
+// Implementations call dedicated STT endpoints (e.g. OpenAI /audio/transcriptions)
+// and return deterministic transcription output. For multimodal LLMs that
+// happen to accept audio input, see internal/ai/audiollm.
+package stt
+
+import (
+	"context"
+	"io"
+)
+
+// Transcriber transcribes audio to text using a provider's dedicated STT endpoint.
+type Transcriber interface {
+	Transcribe(ctx context.Context, req Request) (*Response, error)
+}
+
+// Request is the input to a transcription call.
+type Request struct {
+	Audio       io.Reader
+	Size        int64
+	Filename    string
+	ContentType string // IANA media type, e.g. "audio/wav"
+	Model       string // provider-specific model id (e.g. "whisper-1", "gpt-4o-transcribe")
+	Prompt      string // soft spelling/vocabulary hint (Whisper "prompt" parameter)
+	Language    string // ISO 639-1, optional
+}
+
+// Response is the output of a transcription call.
+type Response struct {
+	Text     string
+	Language string    // empty if provider did not return it
+	Segments []Segment // empty unless provider returned timestamps
+}
+
+// Segment is a timestamped portion of the transcript.
+type Segment struct {
+	Text    string
+	Start   float64
+	End     float64
+	Speaker string // empty unless using a diarization-capable model
+}