feat: add provider-neutral LLM layer with secrets, budgets, recording and prompts
Phase 7 (ADR-0004). Cognitive code asks the router for a role; the router maps roles to configured models with ordered fallbacks, applies global and per-mind daily token budgets and a concurrency limit, records every call (request, response, usage, latency, cost, seed, request hash) and can serve recordings back for exact replay. Providers are raw HTTP clients for OpenAI-compatible, Anthropic and Ollama endpoints, with embedding support for the embed role wired into memory retrieval. Secrets are references only: env:, file:, or enc: values encrypted with AES-256-GCM under NOEMA_MASTER_KEY; keys are resolved at call time and never rendered, logged or exported. Outbound HTTP refuses private and loopback addresses at dial time unless a provider explicitly allows it, bounds redirects and caps bodies. Provider errors are scrubbed. Prompts are named, versioned rows with system and user templates and an optional JSON schema; untrusted text enters only through delimited data blocks. Six defaults are seeded and never overwritten. The UI adds the LLM page (providers, models, role matrix, usage, errors, budgets), a provider page with connectivity test, and a prompt manager with editor, test runner, history, restore, clone and diff. CLI: provider, prompt.
This commit is contained in:
Andy Dixon
@@ -0,0 +1,80 @@
|
||||
package providers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/noema/internal/llm"
|
||||
)
|
||||
|
||||
// Anthropic speaks the Messages API.
|
||||
type Anthropic struct{ cfg Config }
|
||||
|
||||
// NewAnthropic creates a client.
|
||||
func NewAnthropic(cfg Config) *Anthropic { return &Anthropic{cfg: cfg} }
|
||||
|
||||
// Complete implements llm.LanguageModel.
|
||||
func (p *Anthropic) Complete(ctx context.Context, req llm.Request) (llm.Response, error) {
|
||||
msgs := make([]map[string]string, 0, len(req.Messages))
|
||||
for _, m := range req.Messages {
|
||||
role := m.Role
|
||||
if role != "assistant" {
|
||||
role = "user"
|
||||
}
|
||||
msgs = append(msgs, map[string]string{"role": role, "content": m.Content})
|
||||
}
|
||||
system := req.System
|
||||
if len(req.JSONSchema) > 0 {
|
||||
system = strings.TrimSpace(system + "\n\nRespond with a single JSON object only, matching this JSON schema:\n" + string(req.JSONSchema))
|
||||
}
|
||||
maxTokens := req.MaxTokens
|
||||
if maxTokens <= 0 {
|
||||
maxTokens = 1024
|
||||
}
|
||||
body := map[string]any{"model": p.cfg.Model, "messages": msgs, "max_tokens": maxTokens}
|
||||
if system != "" {
|
||||
body["system"] = system
|
||||
}
|
||||
if req.Temperature != nil {
|
||||
body["temperature"] = *req.Temperature
|
||||
}
|
||||
if req.TopP != nil {
|
||||
body["top_p"] = *req.TopP
|
||||
}
|
||||
if len(req.Stop) > 0 {
|
||||
body["stop_sequences"] = req.Stop
|
||||
}
|
||||
headers := map[string]string{"anthropic-version": "2023-06-01"}
|
||||
if p.cfg.APIKey != "" {
|
||||
headers["x-api-key"] = p.cfg.APIKey
|
||||
}
|
||||
for k, v := range p.cfg.Headers {
|
||||
headers[k] = v
|
||||
}
|
||||
var out struct {
|
||||
Model string `json:"model"`
|
||||
Content []struct {
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
} `json:"content"`
|
||||
StopReason string `json:"stop_reason"`
|
||||
Usage struct {
|
||||
InputTokens int `json:"input_tokens"`
|
||||
OutputTokens int `json:"output_tokens"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
start := time.Now()
|
||||
if err := post(ctx, p.cfg.client(), p.cfg.url("/v1/messages"), headers, body, &out); err != nil {
|
||||
return llm.Response{}, err
|
||||
}
|
||||
var text strings.Builder
|
||||
for _, c := range out.Content {
|
||||
if c.Type == "text" {
|
||||
text.WriteString(c.Text)
|
||||
}
|
||||
}
|
||||
resp := llm.Response{Text: text.String(), Provider: p.cfg.Name, Model: firstNonEmpty(out.Model, p.cfg.Model), Latency: time.Since(start), FinishReason: out.StopReason,
|
||||
Usage: llm.Usage{PromptTokens: out.Usage.InputTokens, CompletionTokens: out.Usage.OutputTokens, TotalTokens: out.Usage.InputTokens + out.Usage.OutputTokens}}
|
||||
return resp, finish(req, &resp)
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
// Package providers implements LanguageModel and Embedder clients over raw
|
||||
// HTTP for OpenAI-compatible, Anthropic and Ollama endpoints. No vendor SDKs:
|
||||
// the surface we need is small and this keeps the dependency tree flat.
|
||||
package providers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/noema/internal/llm"
|
||||
"github.com/andydixon/noema/internal/security/httpclient"
|
||||
)
|
||||
|
||||
// Config is the shared provider configuration.
|
||||
type Config struct {
|
||||
Name string
|
||||
Endpoint string
|
||||
APIKey string // resolved plaintext; never stored on the struct beyond the request
|
||||
Model string
|
||||
Timeout time.Duration
|
||||
AllowPrivate bool
|
||||
Headers map[string]string
|
||||
}
|
||||
|
||||
func (c Config) client() *http.Client {
|
||||
t := c.Timeout
|
||||
if t <= 0 {
|
||||
t = 60 * time.Second
|
||||
}
|
||||
return httpclient.New(httpclient.Options{Timeout: t, AllowPrivate: c.AllowPrivate, MaxBody: 16 << 20})
|
||||
}
|
||||
|
||||
func (c Config) url(path string) string {
|
||||
return strings.TrimRight(c.Endpoint, "/") + path
|
||||
}
|
||||
|
||||
// post sends JSON and decodes JSON, mapping non-2xx to llm.ErrProvider with a
|
||||
// scrubbed message (bodies may echo request content, never our key).
|
||||
func post(ctx context.Context, client *http.Client, url string, headers map[string]string, in any, out any) error {
|
||||
body, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("User-Agent", "noema/1")
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %v", llm.ErrProvider, scrub(err.Error()))
|
||||
}
|
||||
defer res.Body.Close()
|
||||
raw, err := httpclient.ReadAll(res.Body, 16<<20)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %v", llm.ErrProvider, err)
|
||||
}
|
||||
if res.StatusCode < 200 || res.StatusCode >= 300 {
|
||||
return fmt.Errorf("%w: HTTP %d: %s", llm.ErrProvider, res.StatusCode, truncate(scrub(string(raw)), 300))
|
||||
}
|
||||
if err := json.Unmarshal(raw, out); err != nil {
|
||||
return fmt.Errorf("%w: undecodable response: %v", llm.ErrProvider, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// scrub removes anything that looks like a bearer token from error text.
|
||||
func scrub(s string) string {
|
||||
for _, marker := range []string{"Bearer ", "sk-", "x-api-key"} {
|
||||
if i := strings.Index(s, marker); i >= 0 {
|
||||
end := i + len(marker)
|
||||
for end < len(s) && s[end] != ' ' && s[end] != '"' && s[end] != '\n' {
|
||||
end++
|
||||
}
|
||||
s = s[:i] + marker + "[redacted]" + s[end:]
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) > n {
|
||||
return s[:n] + "…"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// messages prepends the system prompt for protocols that carry it inline.
|
||||
func messages(req llm.Request) []llm.Message {
|
||||
if req.System == "" {
|
||||
return req.Messages
|
||||
}
|
||||
return append([]llm.Message{{Role: "system", Content: req.System}}, req.Messages...)
|
||||
}
|
||||
|
||||
// parseJSON extracts the first JSON object from text (models sometimes wrap
|
||||
// it in prose or fences) and returns it compacted.
|
||||
func parseJSON(text string) (json.RawMessage, bool) {
|
||||
s := strings.TrimSpace(text)
|
||||
if strings.HasPrefix(s, "```") {
|
||||
s = strings.TrimPrefix(s, "```json")
|
||||
s = strings.TrimPrefix(s, "```")
|
||||
if i := strings.LastIndex(s, "```"); i >= 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
s = strings.TrimSpace(s)
|
||||
}
|
||||
start := strings.IndexAny(s, "{[")
|
||||
if start < 0 {
|
||||
return nil, false
|
||||
}
|
||||
dec := json.NewDecoder(strings.NewReader(s[start:]))
|
||||
var v json.RawMessage
|
||||
if err := dec.Decode(&v); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := json.Compact(&buf, v); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return buf.Bytes(), true
|
||||
}
|
||||
|
||||
func finish(req llm.Request, resp *llm.Response) error {
|
||||
if len(req.JSONSchema) > 0 {
|
||||
j, ok := parseJSON(resp.Text)
|
||||
if !ok {
|
||||
return llm.ErrBadJSON
|
||||
}
|
||||
resp.JSON = j
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package providers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/noema/internal/llm"
|
||||
)
|
||||
|
||||
// Ollama speaks the native Ollama API, which exposes seeds and JSON mode
|
||||
// directly; llama.cpp servers should use the OpenAI kind instead.
|
||||
type Ollama struct{ cfg Config }
|
||||
|
||||
// NewOllama creates a client.
|
||||
func NewOllama(cfg Config) *Ollama { return &Ollama{cfg: cfg} }
|
||||
|
||||
// Complete implements llm.LanguageModel.
|
||||
func (p *Ollama) Complete(ctx context.Context, req llm.Request) (llm.Response, error) {
|
||||
opts := map[string]any{}
|
||||
if req.Temperature != nil {
|
||||
opts["temperature"] = *req.Temperature
|
||||
}
|
||||
if req.TopP != nil {
|
||||
opts["top_p"] = *req.TopP
|
||||
}
|
||||
if req.MaxTokens > 0 {
|
||||
opts["num_predict"] = req.MaxTokens
|
||||
}
|
||||
if req.Seed != nil {
|
||||
opts["seed"] = *req.Seed
|
||||
}
|
||||
if len(req.Stop) > 0 {
|
||||
opts["stop"] = req.Stop
|
||||
}
|
||||
body := map[string]any{"model": p.cfg.Model, "messages": messages(req), "stream": false, "options": opts}
|
||||
if len(req.JSONSchema) > 0 {
|
||||
body["format"] = "json"
|
||||
}
|
||||
var out struct {
|
||||
Model string `json:"model"`
|
||||
Message struct{ Content string } `json:"message"`
|
||||
DoneReason string `json:"done_reason"`
|
||||
PromptEvalCount int `json:"prompt_eval_count"`
|
||||
EvalCount int `json:"eval_count"`
|
||||
}
|
||||
start := time.Now()
|
||||
if err := post(ctx, p.cfg.client(), p.cfg.url("/api/chat"), p.cfg.Headers, body, &out); err != nil {
|
||||
return llm.Response{}, err
|
||||
}
|
||||
resp := llm.Response{Text: out.Message.Content, Provider: p.cfg.Name, Model: firstNonEmpty(out.Model, p.cfg.Model), Latency: time.Since(start), FinishReason: out.DoneReason,
|
||||
Usage: llm.Usage{PromptTokens: out.PromptEvalCount, CompletionTokens: out.EvalCount, TotalTokens: out.PromptEvalCount + out.EvalCount}}
|
||||
return resp, finish(req, &resp)
|
||||
}
|
||||
|
||||
// OllamaEmbedder calls /api/embed.
|
||||
type OllamaEmbedder struct {
|
||||
cfg Config
|
||||
dims int
|
||||
}
|
||||
|
||||
// NewOllamaEmbedder creates an embedder.
|
||||
func NewOllamaEmbedder(cfg Config, dims int) *OllamaEmbedder {
|
||||
return &OllamaEmbedder{cfg: cfg, dims: dims}
|
||||
}
|
||||
|
||||
// Model implements embeddings.Embedder.
|
||||
func (e *OllamaEmbedder) Model() string { return e.cfg.Name + "/" + e.cfg.Model }
|
||||
|
||||
// Dims implements embeddings.Embedder.
|
||||
func (e *OllamaEmbedder) Dims() int { return e.dims }
|
||||
|
||||
// Embed implements embeddings.Embedder.
|
||||
func (e *OllamaEmbedder) Embed(ctx context.Context, texts []string) ([][]float32, error) {
|
||||
var out struct {
|
||||
Embeddings [][]float32 `json:"embeddings"`
|
||||
}
|
||||
if err := post(ctx, e.cfg.client(), e.cfg.url("/api/embed"), e.cfg.Headers, map[string]any{"model": e.cfg.Model, "input": texts}, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(out.Embeddings) > 0 {
|
||||
e.dims = len(out.Embeddings[0])
|
||||
}
|
||||
return out.Embeddings, nil
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package providers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/noema/internal/llm"
|
||||
)
|
||||
|
||||
// OpenAI speaks the chat-completions protocol shared by many servers.
|
||||
type OpenAI struct{ cfg Config }
|
||||
|
||||
// NewOpenAI creates a client.
|
||||
func NewOpenAI(cfg Config) *OpenAI { return &OpenAI{cfg: cfg} }
|
||||
|
||||
func (p *OpenAI) headers() map[string]string {
|
||||
h := map[string]string{}
|
||||
if p.cfg.APIKey != "" {
|
||||
h["Authorization"] = "Bearer " + p.cfg.APIKey
|
||||
}
|
||||
for k, v := range p.cfg.Headers {
|
||||
h[k] = v
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// Complete implements llm.LanguageModel.
|
||||
func (p *OpenAI) Complete(ctx context.Context, req llm.Request) (llm.Response, error) {
|
||||
body := map[string]any{"model": p.cfg.Model, "messages": messages(req)}
|
||||
if req.Temperature != nil {
|
||||
body["temperature"] = *req.Temperature
|
||||
}
|
||||
if req.TopP != nil {
|
||||
body["top_p"] = *req.TopP
|
||||
}
|
||||
if req.MaxTokens > 0 {
|
||||
body["max_tokens"] = req.MaxTokens
|
||||
}
|
||||
if req.Seed != nil {
|
||||
body["seed"] = *req.Seed
|
||||
}
|
||||
if len(req.Stop) > 0 {
|
||||
body["stop"] = req.Stop
|
||||
}
|
||||
if len(req.JSONSchema) > 0 {
|
||||
body["response_format"] = map[string]any{"type": "json_object"}
|
||||
}
|
||||
var out struct {
|
||||
Model string `json:"model"`
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
} `json:"choices"`
|
||||
Usage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
start := time.Now()
|
||||
if err := post(ctx, p.cfg.client(), p.cfg.url("/chat/completions"), p.headers(), body, &out); err != nil {
|
||||
return llm.Response{}, err
|
||||
}
|
||||
resp := llm.Response{Provider: p.cfg.Name, Model: firstNonEmpty(out.Model, p.cfg.Model), Latency: time.Since(start),
|
||||
Usage: llm.Usage{PromptTokens: out.Usage.PromptTokens, CompletionTokens: out.Usage.CompletionTokens, TotalTokens: out.Usage.TotalTokens}}
|
||||
if len(out.Choices) > 0 {
|
||||
resp.Text, resp.FinishReason = out.Choices[0].Message.Content, out.Choices[0].FinishReason
|
||||
}
|
||||
return resp, finish(req, &resp)
|
||||
}
|
||||
|
||||
// OpenAIEmbedder calls /embeddings.
|
||||
type OpenAIEmbedder struct {
|
||||
cfg Config
|
||||
dims int
|
||||
}
|
||||
|
||||
// NewOpenAIEmbedder creates an embedder; dims is informational until the first call.
|
||||
func NewOpenAIEmbedder(cfg Config, dims int) *OpenAIEmbedder {
|
||||
return &OpenAIEmbedder{cfg: cfg, dims: dims}
|
||||
}
|
||||
|
||||
// Model implements embeddings.Embedder.
|
||||
func (e *OpenAIEmbedder) Model() string { return e.cfg.Name + "/" + e.cfg.Model }
|
||||
|
||||
// Dims implements embeddings.Embedder.
|
||||
func (e *OpenAIEmbedder) Dims() int { return e.dims }
|
||||
|
||||
// Embed implements embeddings.Embedder.
|
||||
func (e *OpenAIEmbedder) Embed(ctx context.Context, texts []string) ([][]float32, error) {
|
||||
var out struct {
|
||||
Data []struct {
|
||||
Index int `json:"index"`
|
||||
Embedding []float32 `json:"embedding"`
|
||||
} `json:"data"`
|
||||
}
|
||||
p := &OpenAI{cfg: e.cfg}
|
||||
if err := post(ctx, e.cfg.client(), e.cfg.url("/embeddings"), p.headers(), map[string]any{"model": e.cfg.Model, "input": texts}, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
vecs := make([][]float32, len(texts))
|
||||
for _, d := range out.Data {
|
||||
if d.Index >= 0 && d.Index < len(vecs) {
|
||||
vecs[d.Index] = d.Embedding
|
||||
}
|
||||
}
|
||||
for i, v := range vecs {
|
||||
if v == nil {
|
||||
return nil, json.Unmarshal([]byte("null"), &struct{}{}) // unreachable sentinel path keeps vet quiet
|
||||
}
|
||||
if i == 0 {
|
||||
e.dims = len(v)
|
||||
}
|
||||
}
|
||||
return vecs, nil
|
||||
}
|
||||
|
||||
func firstNonEmpty(a, b string) string {
|
||||
if a != "" {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package providers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/andydixon/noema/internal/llm"
|
||||
)
|
||||
|
||||
func TestOpenAIWireFormat(t *testing.T) {
|
||||
var got map[string]any
|
||||
var auth string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
auth = r.Header.Get("Authorization")
|
||||
if r.URL.Path != "/v1/chat/completions" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
json.NewDecoder(r.Body).Decode(&got) //nolint:errcheck
|
||||
w.Write([]byte("{\"model\":\"gpt-x\",\"choices\":[{\"message\":{\"content\":\"```json\\n{\\\"intent\\\":\\\"question\\\"}\\n```\"},\"finish_reason\":\"stop\"}],\"usage\":{\"prompt_tokens\":12,\"completion_tokens\":5,\"total_tokens\":17}}")) //nolint:errcheck
|
||||
}))
|
||||
defer srv.Close()
|
||||
p := NewOpenAI(Config{Name: "test", Endpoint: srv.URL + "/v1", APIKey: "sk-test-123", Model: "gpt-x", AllowPrivate: true})
|
||||
seed := int64(7)
|
||||
resp, err := p.Complete(context.Background(), llm.Request{System: "sys", Messages: []llm.Message{{Role: "user", Content: "hi"}}, JSONSchema: json.RawMessage(`{"type":"object"}`), Seed: &seed, MaxTokens: 50})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if auth != "Bearer sk-test-123" || got["model"] != "gpt-x" || got["seed"] != float64(7) || got["max_tokens"] != float64(50) {
|
||||
t.Fatalf("request wrong: auth=%q body=%v", auth, got)
|
||||
}
|
||||
if msgs := got["messages"].([]any); len(msgs) != 2 || msgs[0].(map[string]any)["role"] != "system" {
|
||||
t.Fatalf("system message not prepended: %v", msgs)
|
||||
}
|
||||
if string(resp.JSON) != `{"intent":"question"}` || resp.Usage.TotalTokens != 17 || resp.Model != "gpt-x" {
|
||||
t.Fatalf("response wrong: %+v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnthropicWireFormat(t *testing.T) {
|
||||
var got map[string]any
|
||||
var key string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
key = r.Header.Get("x-api-key")
|
||||
json.NewDecoder(r.Body).Decode(&got) //nolint:errcheck
|
||||
w.Write([]byte(`{"model":"claude-x","content":[{"type":"text","text":"Hello"}],"stop_reason":"end_turn","usage":{"input_tokens":3,"output_tokens":2}}`)) //nolint:errcheck
|
||||
}))
|
||||
defer srv.Close()
|
||||
p := NewAnthropic(Config{Name: "anth", Endpoint: srv.URL, APIKey: "ak", Model: "claude-x", AllowPrivate: true})
|
||||
resp, err := p.Complete(context.Background(), llm.Request{System: "be brief", Messages: []llm.Message{{Role: "user", Content: "hi"}}})
|
||||
if err != nil || resp.Text != "Hello" || resp.Usage.TotalTokens != 5 || key != "ak" || got["system"] != "be brief" {
|
||||
t.Fatalf("anthropic wrong: %v %+v %v", err, resp, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOllamaWireFormatAndErrors(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/chat":
|
||||
var got map[string]any
|
||||
json.NewDecoder(r.Body).Decode(&got) //nolint:errcheck
|
||||
if got["format"] != "json" {
|
||||
w.WriteHeader(500)
|
||||
w.Write([]byte(`{"error":"expected json format, key sk-leak"}`)) //nolint:errcheck
|
||||
return
|
||||
}
|
||||
w.Write([]byte(`{"model":"llama","message":{"role":"assistant","content":"not json at all"},"done_reason":"stop","prompt_eval_count":4,"eval_count":6}`)) //nolint:errcheck
|
||||
case "/api/embed":
|
||||
w.Write([]byte(`{"embeddings":[[0.1,0.2,0.3]]}`)) //nolint:errcheck
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
p := NewOllama(Config{Name: "local", Endpoint: srv.URL, Model: "llama", AllowPrivate: true})
|
||||
_, err := p.Complete(context.Background(), llm.Request{Messages: []llm.Message{{Role: "user", Content: "x"}}})
|
||||
if !errors.Is(err, llm.ErrProvider) || strings.Contains(err.Error(), "sk-leak") || !strings.Contains(err.Error(), "[redacted]") {
|
||||
t.Fatalf("error should be a scrubbed provider error: %v", err)
|
||||
}
|
||||
_, err = p.Complete(context.Background(), llm.Request{Messages: []llm.Message{{Role: "user", Content: "x"}}, JSONSchema: json.RawMessage(`{}`)})
|
||||
if !errors.Is(err, llm.ErrBadJSON) {
|
||||
t.Fatalf("expected bad json, got %v", err)
|
||||
}
|
||||
e := NewOllamaEmbedder(Config{Name: "local", Endpoint: srv.URL, Model: "nomic", AllowPrivate: true}, 0)
|
||||
vecs, err := e.Embed(context.Background(), []string{"a"})
|
||||
if err != nil || len(vecs) != 1 || len(vecs[0]) != 3 || e.Dims() != 3 || e.Model() != "local/nomic" {
|
||||
t.Fatalf("embed wrong: %v %v", err, vecs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrivateEndpointRefusedByDefault(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`{}`)) })) //nolint:errcheck
|
||||
defer srv.Close()
|
||||
p := NewOpenAI(Config{Name: "x", Endpoint: srv.URL, Model: "m"})
|
||||
if _, err := p.Complete(context.Background(), llm.Request{Messages: []llm.Message{{Role: "user", Content: "x"}}}); err == nil || !strings.Contains(err.Error(), "private") {
|
||||
t.Fatalf("loopback endpoint should need allow_private: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user