import { Injectable, Logger } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; import * as fs from "fs/promises"; import * as path from "path"; import { SecretPattern, SecretMatch, SecretScanResult, SecretScannerConfig } from "./types"; /** * Service for scanning files and content for secrets */ @Injectable() export class SecretScannerService { private readonly logger = new Logger(SecretScannerService.name); private readonly patterns: SecretPattern[]; private readonly config: SecretScannerConfig; // Whitelist patterns - these are placeholder patterns, not actual secrets private readonly whitelistPatterns = [ /your-.*-here/i, /^xxxx+$/i, /^\*\*\*\*+$/i, /^example$/i, // Just the word "example" alone /placeholder/i, /change-me/i, /replace-me/i, /^<.*>$/, // /^\$\{.*\}$/, // ${YOUR_KEY} /test/i, // "test" indicator /sample/i, // "sample" indicator /demo/i, // "demo" indicator /^xxxx.*xxxx$/i, // multiple xxxx pattern ]; constructor(private readonly configService: ConfigService) { this.config = { customPatterns: this.configService.get("orchestrator.secretScanner.customPatterns") ?? [], excludePatterns: this.configService.get("orchestrator.secretScanner.excludePatterns") ?? [], scanBinaryFiles: this.configService.get("orchestrator.secretScanner.scanBinaryFiles") ?? false, maxFileSize: this.configService.get("orchestrator.secretScanner.maxFileSize") ?? 10 * 1024 * 1024, // 10MB default }; this.patterns = this.loadPatterns(); } /** * Load built-in and custom secret patterns */ private loadPatterns(): SecretPattern[] { const builtInPatterns: SecretPattern[] = [ { name: "AWS Access Key", pattern: /AKIA[0-9A-Z]{16}/g, description: "AWS Access Key ID", severity: "critical", }, { name: "Claude API Key", pattern: /sk-ant-[a-zA-Z0-9\-_]{40,}/g, description: "Anthropic Claude API Key", severity: "critical", }, { name: "Generic API Key", pattern: /api[_-]?key\s*[:=]\s*['"]?[a-zA-Z0-9]{10,}['"]?/gi, description: "Generic API Key", severity: "high", }, { name: "Password Assignment", pattern: /password\s*[:=]\s*['"]?[a-zA-Z0-9!@#$%^&*]{8,}['"]?/gi, description: "Password in code", severity: "high", }, { name: "Private Key", pattern: /-----BEGIN[\s\w]*PRIVATE KEY-----/g, description: "Private cryptographic key", severity: "critical", }, { name: "JWT Token", pattern: /eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g, description: "JSON Web Token", severity: "high", }, { name: "Bearer Token", pattern: /Bearer\s+[A-Za-z0-9\-._~+/]+=*/g, description: "Bearer authentication token", severity: "high", }, { name: "Generic Secret", pattern: /secret\s*[:=]\s*['"]?[a-zA-Z0-9]{16,}['"]?/gi, description: "Generic secret value", severity: "medium", }, ]; // Add custom patterns from config return [...builtInPatterns, ...(this.config.customPatterns ?? [])]; } /** * Check if a match should be whitelisted */ private isWhitelisted(match: string, filePath?: string): boolean { // Extract the value part from patterns like 'api_key="value"' or 'password=value' // This regex extracts quoted or unquoted values after = or : const valueMatch = /[:=]\s*['"]?([^'"\s]+)['"]?$/.exec(match); const value = valueMatch ? valueMatch[1] : match; // Check if it's an AWS example key specifically // AWS documentation uses keys like AKIAIOSFODNN7EXAMPLE, AKIATESTSAMPLE, etc. if (value.startsWith("AKIA") && /EXAMPLE|SAMPLE|TEST|DEMO/i.test(value)) { return true; } // AWS EXAMPLE keys are documented examples, not real secrets // But we still want to catch them unless in .example files const isExampleFile = filePath && (path.basename(filePath).toLowerCase().includes(".example") || path.basename(filePath).toLowerCase().includes("sample") || path.basename(filePath).toLowerCase().includes("template")); // Only whitelist obvious placeholders const isObviousPlaceholder = this.whitelistPatterns.some((pattern) => pattern.test(value)); // If it's an example file AND has placeholder text, whitelist it if (isExampleFile && isObviousPlaceholder) { return true; } // Otherwise, whitelist if it's an obvious placeholder if (isObviousPlaceholder) { return true; } return false; } /** * Match a single pattern against content */ private matchPattern(content: string, pattern: SecretPattern, filePath?: string): SecretMatch[] { const matches: SecretMatch[] = []; const lines = content.split("\n"); // Reset regex lastIndex to ensure clean matching pattern.pattern.lastIndex = 0; for (let lineIndex = 0; lineIndex < lines.length; lineIndex++) { const line = lines[lineIndex]; const lineNumber = lineIndex + 1; // Create a new regex from the pattern to avoid state issues // eslint-disable-next-line security/detect-non-literal-regexp -- Pattern source comes from validated config, not user input const regex = new RegExp(pattern.pattern.source, pattern.pattern.flags); let regexMatch: RegExpExecArray | null; while ((regexMatch = regex.exec(line)) !== null) { const matchText = regexMatch[0]; // Skip if whitelisted if (this.isWhitelisted(matchText, filePath)) { continue; } matches.push({ patternName: pattern.name, match: matchText, line: lineNumber, column: regexMatch.index + 1, severity: pattern.severity, context: line.trim(), }); // Prevent infinite loops on zero-width matches if (regexMatch.index === regex.lastIndex) { regex.lastIndex++; } } } return matches; } /** * Scan content for secrets */ scanContent(content: string, filePath?: string): SecretScanResult { const allMatches: SecretMatch[] = []; // Scan with each pattern for (const pattern of this.patterns) { const matches = this.matchPattern(content, pattern, filePath); allMatches.push(...matches); } return { filePath, hasSecrets: allMatches.length > 0, matches: allMatches, count: allMatches.length, scannedSuccessfully: true, }; } /** * Scan a file for secrets */ async scanFile(filePath: string): Promise { try { // Check if file should be excluded const fileName = path.basename(filePath); for (const excludePattern of this.config.excludePatterns ?? []) { // Convert glob pattern to regex if needed const pattern = typeof excludePattern === "string" ? excludePattern.replace(/\./g, "\\.").replace(/\*/g, ".*") : excludePattern; if (fileName.match(pattern)) { this.logger.debug(`Skipping excluded file: ${filePath}`); return { filePath, hasSecrets: false, matches: [], count: 0, scannedSuccessfully: true, }; } } // Check file size // eslint-disable-next-line security/detect-non-literal-fs-filename -- Scanner must access arbitrary files by design const stats = await fs.stat(filePath); if (this.config.maxFileSize && stats.size > this.config.maxFileSize) { this.logger.warn( `File ${filePath} exceeds max size (${stats.size.toString()} bytes), skipping` ); return { filePath, hasSecrets: false, matches: [], count: 0, scannedSuccessfully: true, }; } // Read file content // eslint-disable-next-line security/detect-non-literal-fs-filename -- Scanner must access arbitrary files by design const content = await fs.readFile(filePath, "utf-8"); // Scan content return this.scanContent(content, filePath); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); this.logger.warn(`Failed to scan file ${filePath}: ${errorMessage}`); // Return error state - file could not be scanned, NOT clean return { filePath, hasSecrets: false, matches: [], count: 0, scannedSuccessfully: false, scanError: errorMessage, }; } } /** * Scan multiple files for secrets */ async scanFiles(filePaths: string[]): Promise { const results: SecretScanResult[] = []; for (const filePath of filePaths) { const result = await this.scanFile(filePath); results.push(result); } return results; } /** * Get a summary of scan results */ getScanSummary(results: SecretScanResult[]): { totalFiles: number; filesWithSecrets: number; totalSecrets: number; filesWithErrors: number; bySeverity: Record; } { const summary = { totalFiles: results.length, filesWithSecrets: results.filter((r) => r.hasSecrets).length, totalSecrets: results.reduce((sum, r) => sum + r.count, 0), filesWithErrors: results.filter((r) => !r.scannedSuccessfully).length, bySeverity: { critical: 0, high: 0, medium: 0, low: 0, }, }; for (const result of results) { for (const match of result.matches) { summary.bySeverity[match.severity]++; } } return summary; } }