/** * Provider Adapter Integration Tests — M3-012 * * Verifies that all five provider adapters (Anthropic, OpenAI, OpenRouter, Z.ai, Ollama) * are properly integrated: registration, model listing, graceful degradation without * API keys, capability matrix correctness, and ProviderCredentialsService behaviour. * * These tests are designed to run in CI with no real API keys; they test graceful * degradation and static configuration rather than live network calls. */ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import { ModelRegistry, AuthStorage } from '@mariozechner/pi-coding-agent'; import { AnthropicAdapter } from '../adapters/anthropic.adapter.js'; import { OpenAIAdapter } from '../adapters/openai.adapter.js'; import { OpenRouterAdapter } from '../adapters/openrouter.adapter.js'; import { ZaiAdapter } from '../adapters/zai.adapter.js'; import { OllamaAdapter } from '../adapters/ollama.adapter.js'; import { ProviderService } from '../provider.service.js'; import { getModelCapability, MODEL_CAPABILITIES, findModelsByCapability, } from '../model-capabilities.js'; // --------------------------------------------------------------------------- // Environment helpers // --------------------------------------------------------------------------- const ALL_PROVIDER_KEYS = [ 'ANTHROPIC_API_KEY', 'OPENAI_API_KEY', 'OPENROUTER_API_KEY', 'ZAI_API_KEY', 'ZAI_BASE_URL', 'OLLAMA_BASE_URL', 'OLLAMA_HOST', 'OLLAMA_MODELS', 'BETTER_AUTH_SECRET', ] as const; type EnvKey = (typeof ALL_PROVIDER_KEYS)[number]; function saveAndClearEnv(): Map { const saved = new Map(); for (const key of ALL_PROVIDER_KEYS) { saved.set(key, process.env[key]); delete process.env[key]; } return saved; } function restoreEnv(saved: Map): void { for (const key of ALL_PROVIDER_KEYS) { const value = saved.get(key); if (value === undefined) { delete process.env[key]; } else { process.env[key] = value; } } } function makeRegistry(): ModelRegistry { return ModelRegistry.inMemory(AuthStorage.inMemory()); } // --------------------------------------------------------------------------- // 1. Adapter registration tests // --------------------------------------------------------------------------- describe('AnthropicAdapter', () => { let savedEnv: Map; beforeEach(() => { savedEnv = saveAndClearEnv(); }); afterEach(() => { restoreEnv(savedEnv); }); it('skips registration gracefully when ANTHROPIC_API_KEY is missing', async () => { const adapter = new AnthropicAdapter(makeRegistry()); await expect(adapter.register()).resolves.toBeUndefined(); expect(adapter.listModels()).toEqual([]); }); it('registers and listModels returns expected models when ANTHROPIC_API_KEY is set', async () => { process.env['ANTHROPIC_API_KEY'] = 'sk-ant-test'; const adapter = new AnthropicAdapter(makeRegistry()); await adapter.register(); const models = adapter.listModels(); expect(models.length).toBeGreaterThan(0); const ids = models.map((m) => m.id); expect(ids).toContain('claude-opus-4-6'); expect(ids).toContain('claude-sonnet-4-6'); expect(ids).toContain('claude-haiku-4-5'); for (const model of models) { expect(model.provider).toBe('anthropic'); expect(model.contextWindow).toBe(200000); } }); it('healthCheck returns down with error when ANTHROPIC_API_KEY is missing', async () => { const adapter = new AnthropicAdapter(makeRegistry()); const health = await adapter.healthCheck(); expect(health.status).toBe('down'); expect(health.error).toMatch(/ANTHROPIC_API_KEY/); expect(health.lastChecked).toBeTruthy(); }); it('adapter name is "anthropic"', () => { expect(new AnthropicAdapter(makeRegistry()).name).toBe('anthropic'); }); }); describe('OpenAIAdapter', () => { let savedEnv: Map; beforeEach(() => { savedEnv = saveAndClearEnv(); }); afterEach(() => { restoreEnv(savedEnv); }); it('skips registration gracefully when OPENAI_API_KEY is missing', async () => { const adapter = new OpenAIAdapter(makeRegistry()); await expect(adapter.register()).resolves.toBeUndefined(); expect(adapter.listModels()).toEqual([]); }); it('registers and listModels returns Codex model when OPENAI_API_KEY is set', async () => { process.env['OPENAI_API_KEY'] = 'sk-openai-test'; const adapter = new OpenAIAdapter(makeRegistry()); await adapter.register(); const models = adapter.listModels(); expect(models.length).toBeGreaterThan(0); const ids = models.map((m) => m.id); expect(ids).toContain(OpenAIAdapter.CODEX_MODEL_ID); const codex = models.find((m) => m.id === OpenAIAdapter.CODEX_MODEL_ID)!; expect(codex.provider).toBe('openai'); expect(codex.contextWindow).toBe(128_000); expect(codex.maxTokens).toBe(16_384); }); it('healthCheck returns down with error when OPENAI_API_KEY is missing', async () => { const adapter = new OpenAIAdapter(makeRegistry()); const health = await adapter.healthCheck(); expect(health.status).toBe('down'); expect(health.error).toMatch(/OPENAI_API_KEY/); }); it('adapter name is "openai"', () => { expect(new OpenAIAdapter(makeRegistry()).name).toBe('openai'); }); }); describe('OpenRouterAdapter', () => { let savedEnv: Map; beforeEach(() => { savedEnv = saveAndClearEnv(); // Prevent real network calls during registration — stub global fetch vi.stubGlobal( 'fetch', vi.fn().mockResolvedValue({ ok: true, json: () => Promise.resolve({ data: [ { id: 'openai/gpt-4o', name: 'GPT-4o', context_length: 128000, top_provider: { max_completion_tokens: 4096 }, pricing: { prompt: '0.000005', completion: '0.000015' }, architecture: { input_modalities: ['text', 'image'] }, }, ], }), }), ); }); afterEach(() => { restoreEnv(savedEnv); vi.unstubAllGlobals(); }); it('skips registration gracefully when OPENROUTER_API_KEY is missing', async () => { vi.unstubAllGlobals(); // no fetch call expected const adapter = new OpenRouterAdapter(); await expect(adapter.register()).resolves.toBeUndefined(); expect(adapter.listModels()).toEqual([]); }); it('registers and listModels returns models when OPENROUTER_API_KEY is set', async () => { process.env['OPENROUTER_API_KEY'] = 'sk-or-test'; const adapter = new OpenRouterAdapter(); await adapter.register(); const models = adapter.listModels(); expect(models.length).toBeGreaterThan(0); const first = models[0]!; expect(first.provider).toBe('openrouter'); expect(first.id).toBe('openai/gpt-4o'); expect(first.inputTypes).toContain('image'); }); it('healthCheck returns down with error when OPENROUTER_API_KEY is missing', async () => { vi.unstubAllGlobals(); // no fetch call expected const adapter = new OpenRouterAdapter(); const health = await adapter.healthCheck(); expect(health.status).toBe('down'); expect(health.error).toMatch(/OPENROUTER_API_KEY/); }); it('continues registration with empty model list when model fetch fails', async () => { process.env['OPENROUTER_API_KEY'] = 'sk-or-test'; vi.stubGlobal( 'fetch', vi.fn().mockResolvedValue({ ok: false, status: 500, }), ); const adapter = new OpenRouterAdapter(); await expect(adapter.register()).resolves.toBeUndefined(); expect(adapter.listModels()).toEqual([]); }); it('adapter name is "openrouter"', () => { expect(new OpenRouterAdapter().name).toBe('openrouter'); }); }); describe('ZaiAdapter', () => { let savedEnv: Map; beforeEach(() => { savedEnv = saveAndClearEnv(); }); afterEach(() => { restoreEnv(savedEnv); }); it('skips registration gracefully when ZAI_API_KEY is missing', async () => { const adapter = new ZaiAdapter(); await expect(adapter.register()).resolves.toBeUndefined(); expect(adapter.listModels()).toEqual([]); }); it('registers and listModels returns glm-5 when ZAI_API_KEY is set', async () => { process.env['ZAI_API_KEY'] = 'zai-test-key'; const adapter = new ZaiAdapter(); await adapter.register(); const models = adapter.listModels(); expect(models.length).toBeGreaterThan(0); const ids = models.map((m) => m.id); expect(ids).toContain('glm-5'); const glm = models.find((m) => m.id === 'glm-5')!; expect(glm.provider).toBe('zai'); }); it('healthCheck returns down with error when ZAI_API_KEY is missing', async () => { const adapter = new ZaiAdapter(); const health = await adapter.healthCheck(); expect(health.status).toBe('down'); expect(health.error).toMatch(/ZAI_API_KEY/); }); it('adapter name is "zai"', () => { expect(new ZaiAdapter().name).toBe('zai'); }); }); describe('OllamaAdapter', () => { let savedEnv: Map; beforeEach(() => { savedEnv = saveAndClearEnv(); }); afterEach(() => { restoreEnv(savedEnv); }); it('skips registration gracefully when OLLAMA_BASE_URL is missing', async () => { const adapter = new OllamaAdapter(makeRegistry()); await expect(adapter.register()).resolves.toBeUndefined(); expect(adapter.listModels()).toEqual([]); }); it('registers via OLLAMA_HOST fallback when OLLAMA_BASE_URL is absent', async () => { process.env['OLLAMA_HOST'] = 'http://localhost:11434'; const adapter = new OllamaAdapter(makeRegistry()); await adapter.register(); const models = adapter.listModels(); expect(models.length).toBeGreaterThan(0); }); it('registers default models (llama3.2, codellama, mistral) + embedding models', async () => { process.env['OLLAMA_BASE_URL'] = 'http://localhost:11434'; const adapter = new OllamaAdapter(makeRegistry()); await adapter.register(); const models = adapter.listModels(); const ids = models.map((m) => m.id); // Default completion models expect(ids).toContain('llama3.2'); expect(ids).toContain('codellama'); expect(ids).toContain('mistral'); // Embedding models expect(ids).toContain('nomic-embed-text'); expect(ids).toContain('mxbai-embed-large'); for (const model of models) { expect(model.provider).toBe('ollama'); } }); it('registers custom OLLAMA_MODELS list', async () => { process.env['OLLAMA_BASE_URL'] = 'http://localhost:11434'; process.env['OLLAMA_MODELS'] = 'phi3,gemma2'; const adapter = new OllamaAdapter(makeRegistry()); await adapter.register(); const completionIds = adapter.listModels().map((m) => m.id); expect(completionIds).toContain('phi3'); expect(completionIds).toContain('gemma2'); expect(completionIds).not.toContain('llama3.2'); }); it('healthCheck returns down with error when OLLAMA_BASE_URL is missing', async () => { const adapter = new OllamaAdapter(makeRegistry()); const health = await adapter.healthCheck(); expect(health.status).toBe('down'); expect(health.error).toMatch(/OLLAMA_BASE_URL/); }); it('adapter name is "ollama"', () => { expect(new OllamaAdapter(makeRegistry()).name).toBe('ollama'); }); }); // --------------------------------------------------------------------------- // 2. ProviderService integration // --------------------------------------------------------------------------- describe('ProviderService — adapter array integration', () => { let savedEnv: Map; beforeEach(() => { savedEnv = saveAndClearEnv(); }); afterEach(() => { restoreEnv(savedEnv); }); it('contains all 5 adapters (ollama, anthropic, openai, openrouter, zai)', async () => { const service = new ProviderService(null); await service.onModuleInit(); // Exercise getAdapter for all five known provider names const expectedProviders = ['ollama', 'anthropic', 'openai', 'openrouter', 'zai']; for (const name of expectedProviders) { const adapter = service.getAdapter(name); expect(adapter, `Expected adapter "${name}" to be registered`).toBeDefined(); expect(adapter!.name).toBe(name); } }); it('healthCheckAll runs without crashing and returns status for all 5 providers', async () => { const service = new ProviderService(null); await service.onModuleInit(); const results = await service.healthCheckAll(); expect(typeof results).toBe('object'); const expectedProviders = ['ollama', 'anthropic', 'openai', 'openrouter', 'zai']; for (const name of expectedProviders) { const health = results[name]; expect(health, `Expected health result for provider "${name}"`).toBeDefined(); expect(['healthy', 'degraded', 'down']).toContain(health!.status); expect(health!.lastChecked).toBeTruthy(); } }); it('healthCheckAll reports "down" for all providers when no keys are set', async () => { const service = new ProviderService(null); await service.onModuleInit(); const results = await service.healthCheckAll(); // All unconfigured providers should be down (not healthy) for (const [, health] of Object.entries(results)) { expect(['down', 'degraded']).toContain(health.status); } }); it('getProvidersHealth returns entries for all 5 providers', async () => { const service = new ProviderService(null); await service.onModuleInit(); const healthList = service.getProvidersHealth(); const names = healthList.map((h) => h.name); for (const expected of ['ollama', 'anthropic', 'openai', 'openrouter', 'zai']) { expect(names).toContain(expected); } for (const entry of healthList) { expect(entry).toHaveProperty('name'); expect(entry).toHaveProperty('status'); expect(entry).toHaveProperty('lastChecked'); expect(typeof entry.modelCount).toBe('number'); } }); it('service initialises without error when all env keys are absent', async () => { const service = new ProviderService(null); await expect(service.onModuleInit()).resolves.toBeUndefined(); service.onModuleDestroy(); }); }); // --------------------------------------------------------------------------- // 3. Model capability matrix // --------------------------------------------------------------------------- describe('Model capability matrix', () => { const expectedModels: Array<{ id: string; provider: string; tier: string; contextWindow: number; reasoning?: boolean; vision?: boolean; embedding?: boolean; }> = [ { id: 'claude-opus-4-6', provider: 'anthropic', tier: 'premium', contextWindow: 200000, reasoning: true, vision: true, }, { id: 'claude-sonnet-4-6', provider: 'anthropic', tier: 'standard', contextWindow: 200000, reasoning: true, vision: true, }, { id: 'claude-haiku-4-5', provider: 'anthropic', tier: 'cheap', contextWindow: 200000, reasoning: false, vision: true, }, { id: 'codex-gpt-5.4', provider: 'openai', tier: 'premium', contextWindow: 128000, }, { id: 'glm-5', provider: 'zai', tier: 'standard', contextWindow: 128000, }, { id: 'llama3.2', provider: 'ollama', tier: 'local', contextWindow: 128000, }, { id: 'codellama', provider: 'ollama', tier: 'local', contextWindow: 16000, }, { id: 'mistral', provider: 'ollama', tier: 'local', contextWindow: 32000, }, { id: 'nomic-embed-text', provider: 'ollama', tier: 'local', contextWindow: 8192, embedding: true, }, { id: 'mxbai-embed-large', provider: 'ollama', tier: 'local', contextWindow: 8192, embedding: true, }, ]; it('MODEL_CAPABILITIES contains all expected model IDs', () => { const allIds = MODEL_CAPABILITIES.map((m) => m.id); for (const { id } of expectedModels) { expect(allIds, `Expected model "${id}" in capability matrix`).toContain(id); } }); it('getModelCapability() returns correct tier and context window for each model', () => { for (const expected of expectedModels) { const cap = getModelCapability(expected.id); expect(cap, `getModelCapability("${expected.id}") should be defined`).toBeDefined(); expect(cap!.provider).toBe(expected.provider); expect(cap!.tier).toBe(expected.tier); expect(cap!.contextWindow).toBe(expected.contextWindow); } }); it('Anthropic models have correct capability flags (tools, streaming, vision, reasoning)', () => { for (const expected of expectedModels.filter((m) => m.provider === 'anthropic')) { const cap = getModelCapability(expected.id)!; expect(cap.capabilities.tools).toBe(true); expect(cap.capabilities.streaming).toBe(true); if (expected.vision !== undefined) { expect(cap.capabilities.vision).toBe(expected.vision); } if (expected.reasoning !== undefined) { expect(cap.capabilities.reasoning).toBe(expected.reasoning); } } }); it('Embedding models have embedding flag=true and other flags=false', () => { for (const expected of expectedModels.filter((m) => m.embedding)) { const cap = getModelCapability(expected.id)!; expect(cap.capabilities.embedding).toBe(true); expect(cap.capabilities.tools).toBe(false); expect(cap.capabilities.streaming).toBe(false); expect(cap.capabilities.reasoning).toBe(false); } }); it('findModelsByCapability filters by tier correctly', () => { const premiumModels = findModelsByCapability({ tier: 'premium' }); expect(premiumModels.length).toBeGreaterThan(0); for (const m of premiumModels) { expect(m.tier).toBe('premium'); } }); it('findModelsByCapability filters by provider correctly', () => { const anthropicModels = findModelsByCapability({ provider: 'anthropic' }); expect(anthropicModels.length).toBe(3); for (const m of anthropicModels) { expect(m.provider).toBe('anthropic'); } }); it('findModelsByCapability filters by capability flags correctly', () => { const reasoningModels = findModelsByCapability({ capabilities: { reasoning: true } }); expect(reasoningModels.length).toBeGreaterThan(0); for (const m of reasoningModels) { expect(m.capabilities.reasoning).toBe(true); } const embeddingModels = findModelsByCapability({ capabilities: { embedding: true } }); expect(embeddingModels.length).toBeGreaterThan(0); for (const m of embeddingModels) { expect(m.capabilities.embedding).toBe(true); } }); it('getModelCapability returns undefined for unknown model IDs', () => { expect(getModelCapability('not-a-real-model')).toBeUndefined(); expect(getModelCapability('')).toBeUndefined(); }); it('all Anthropic models have maxOutputTokens > 0', () => { const anthropicModels = MODEL_CAPABILITIES.filter((m) => m.provider === 'anthropic'); for (const m of anthropicModels) { expect(m.maxOutputTokens).toBeGreaterThan(0); } }); }); // --------------------------------------------------------------------------- // 4. ProviderCredentialsService — unit-level tests (encrypt/decrypt logic) // --------------------------------------------------------------------------- describe('ProviderCredentialsService — encryption helpers', () => { let savedEnv: Map; beforeEach(() => { savedEnv = saveAndClearEnv(); }); afterEach(() => { restoreEnv(savedEnv); }); /** * The service uses module-level functions (encrypt/decrypt) that depend on * BETTER_AUTH_SECRET. We test the behaviour through the service's public API * using an in-memory mock DB so no real Postgres connection is needed. */ it('store/retrieve/remove work correctly with mock DB and BETTER_AUTH_SECRET set', async () => { process.env['BETTER_AUTH_SECRET'] = 'test-secret-for-unit-tests-only'; // Build a minimal in-memory DB mock const rows = new Map< string, { encryptedValue: string; credentialType: string; expiresAt: Date | null; metadata: null; createdAt: Date; updatedAt: Date; } >(); // We import the service but mock its DB dependency manually // by testing the encrypt/decrypt indirectly — using the real module. const { ProviderCredentialsService } = await import('../provider-credentials.service.js'); // Capture stored value from upsert call let storedEncryptedValue = ''; let storedCredentialType = ''; const captureInsert = vi.fn().mockImplementation(() => ({ values: vi .fn() .mockImplementation((data: { encryptedValue: string; credentialType: string }) => { storedEncryptedValue = data.encryptedValue; storedCredentialType = data.credentialType; rows.set('user1:anthropic', { encryptedValue: data.encryptedValue, credentialType: data.credentialType, expiresAt: null, metadata: null, createdAt: new Date(), updatedAt: new Date(), }); return { onConflictDoUpdate: vi.fn().mockResolvedValue(undefined), }; }), })); const captureSelect = vi.fn().mockReturnValue({ from: vi.fn().mockReturnValue({ where: vi.fn().mockReturnValue({ limit: vi.fn().mockImplementation(() => { const row = rows.get('user1:anthropic'); return Promise.resolve(row ? [row] : []); }), }), }), }); const captureDelete = vi.fn().mockReturnValue({ where: vi.fn().mockResolvedValue(undefined), }); const db = { insert: captureInsert, select: captureSelect, delete: captureDelete, }; const service = new ProviderCredentialsService(db as never); // store await service.store('user1', 'anthropic', 'api_key', 'sk-ant-secret-value'); // verify encrypted value is not plain text expect(storedEncryptedValue).not.toBe('sk-ant-secret-value'); expect(storedEncryptedValue.length).toBeGreaterThan(0); expect(storedCredentialType).toBe('api_key'); // retrieve const retrieved = await service.retrieve('user1', 'anthropic'); expect(retrieved).toBe('sk-ant-secret-value'); // remove (clears the row) rows.delete('user1:anthropic'); const afterRemove = await service.retrieve('user1', 'anthropic'); expect(afterRemove).toBeNull(); }); it('retrieve returns null when no credential is stored', async () => { process.env['BETTER_AUTH_SECRET'] = 'test-secret-for-unit-tests-only'; const { ProviderCredentialsService } = await import('../provider-credentials.service.js'); const emptyDb = { select: vi.fn().mockReturnValue({ from: vi.fn().mockReturnValue({ where: vi.fn().mockReturnValue({ limit: vi.fn().mockResolvedValue([]), }), }), }), }; const service = new ProviderCredentialsService(emptyDb as never); const result = await service.retrieve('user-nobody', 'anthropic'); expect(result).toBeNull(); }); it('listProviders returns only metadata, never decrypted values', async () => { process.env['BETTER_AUTH_SECRET'] = 'test-secret-for-unit-tests-only'; const { ProviderCredentialsService } = await import('../provider-credentials.service.js'); const fakeRow = { provider: 'anthropic', credentialType: 'api_key', expiresAt: null, metadata: null, createdAt: new Date(), updatedAt: new Date(), }; const listDb = { select: vi.fn().mockReturnValue({ from: vi.fn().mockReturnValue({ where: vi.fn().mockResolvedValue([fakeRow]), }), }), }; const service = new ProviderCredentialsService(listDb as never); const providers = await service.listProviders('user1'); expect(providers).toHaveLength(1); expect(providers[0]!.provider).toBe('anthropic'); expect(providers[0]!.credentialType).toBe('api_key'); expect(providers[0]!.exists).toBe(true); // Critically: no encrypted or plain-text value is exposed expect(providers[0]).not.toHaveProperty('encryptedValue'); expect(providers[0]).not.toHaveProperty('value'); expect(providers[0]).not.toHaveProperty('apiKey'); }); });