feat(core): add scanner logging and Go-specific exclusions

prosdev · prosdev · commit 1500caf5ff41 · 2025-12-10T13:32:47.000-08:00
Logging:
- Add logger option to ScanOptions and IndexOptions
- Log per-language file counts during discovery
- Log scan progress per language
- Log embedding progress every 10 batches
- Pass logger from CLI (--verbose enables debug level)

Go Exclusions (default):
- *.pb.go (protobuf)
- *.gen.go, *_gen.go (code generators)
- *.pb.gw.go (gRPC gateway)
- mock_*.go (mockgen)
- mocks/ (mock directories)
- testdata/ (test fixtures)

Spinner UX:
- Show 'Embedding X/Y documents' during storing phase
- Added totalDocuments to IndexProgress for accurate counts

Tests updated for new type imports.
diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts
@@ -17,7 +17,7 @@ import { Command } from 'commander';
 import ora from 'ora';
 import { getDefaultConfig, loadConfig } from '../utils/config.js';
 import { formatBytes, getDirectorySize } from '../utils/file.js';
-import { logger } from '../utils/logger.js';
+import { createIndexLogger, logger } from '../utils/logger.js';
 
 /**
  * Check if a command is available
@@ -129,18 +129,27 @@ export const indexCommand = new Command('index')
 
       spinner.text = 'Scanning repository...';
 
+      // Create logger for indexing (verbose mode shows debug logs)
+      const indexLogger = createIndexLogger(options.verbose);
+
       const startTime = Date.now();
       let lastUpdate = startTime;
 
       const stats = await indexer.index({
         force: options.force,
+        logger: indexLogger,
         onProgress: (progress) => {
           const now = Date.now();
           // Update spinner every 100ms to avoid flickering
           if (now - lastUpdate > 100) {
-            const percent = progress.percentComplete || 0;
-            const currentFile = progress.currentFile ? ` ${progress.currentFile}` : '';
-            spinner.text = `${progress.phase}:${currentFile} (${percent.toFixed(0)}%)`;
+            if (progress.phase === 'storing' && progress.totalDocuments) {
+              // Show document count with percentage
+              const pct = Math.round((progress.documentsIndexed / progress.totalDocuments) * 100);
+              spinner.text = `Embedding ${progress.documentsIndexed}/${progress.totalDocuments} documents (${pct}%)`;
+            } else {
+              const percent = progress.percentComplete || 0;
+              spinner.text = `${progress.phase} (${percent.toFixed(0)}%)`;
+            }
             lastUpdate = now;
           }
         },
diff --git a/packages/cli/src/utils/logger.ts b/packages/cli/src/utils/logger.ts
@@ -2,7 +2,7 @@
  * CLI Logger using @lytics/kero
  */
 
-import { createLogger } from '@lytics/kero';
+import { createLogger, type Logger, type LogLevel } from '@lytics/kero';
 
 // Create a logger with pretty output and icons
 const keroLogger = createLogger({
@@ -36,3 +36,14 @@ export const logger = {
     keroLogger.debug(message);
   },
 };
+
+/**
+ * Create a logger for indexing operations with configurable verbosity
+ */
+export function createIndexLogger(verbose: boolean): Logger {
+  const level: LogLevel = verbose ? 'debug' : 'info';
+  return createLogger({
+    level,
+    format: 'pretty',
+  });
+}
diff --git a/packages/core/src/indexer/index.ts b/packages/core/src/indexer/index.ts
@@ -87,12 +87,16 @@ export class RepositoryIndexer {
         include: options.languages?.map((lang) => `**/*.${getExtensionForLanguage(lang)}`),
         exclude: [...this.config.excludePatterns, ...(options.excludePatterns || [])],
         languages: options.languages,
+        logger: options.logger,
       });
 
       filesScanned = scanResult.documents.length;
       documentsExtracted = scanResult.documents.length;
 
       // Phase 2: Prepare documents for embedding
+      const logger = options.logger?.child({ component: 'indexer' });
+      logger?.info({ documents: documentsExtracted }, 'Preparing documents for embedding');
+
       onProgress?.({
         phase: 'embedding',
         filesProcessed: filesScanned,
@@ -104,27 +108,49 @@ export class RepositoryIndexer {
       const embeddingDocuments = prepareDocumentsForEmbedding(scanResult.documents);
 
       // Phase 3: Batch embed and store
+      logger?.info(
+        {
+          documents: embeddingDocuments.length,
+          batchSize: options.batchSize || this.config.batchSize,
+        },
+        'Starting embedding and storage'
+      );
+
       onProgress?.({
         phase: 'storing',
         filesProcessed: filesScanned,
         totalFiles: filesScanned,
         documentsIndexed: 0,
+        totalDocuments: embeddingDocuments.length,
         percentComplete: 66,
       });
 
       const batchSize = options.batchSize || this.config.batchSize;
+      const totalBatches = Math.ceil(embeddingDocuments.length / batchSize);
+      let batchNum = 0;
+
       for (let i = 0; i < embeddingDocuments.length; i += batchSize) {
         const batch = embeddingDocuments.slice(i, i + batchSize);
+        batchNum++;
 
         try {
           await this.vectorStorage.addDocuments(batch);
           documentsIndexed += batch.length;
 
+          // Log progress every 10 batches or on last batch
+          if (batchNum % 10 === 0 || batchNum === totalBatches) {
+            logger?.info(
+              { batch: batchNum, totalBatches, documentsIndexed, total: embeddingDocuments.length },
+              `Embedded ${documentsIndexed}/${embeddingDocuments.length} documents`
+            );
+          }
+
           onProgress?.({
             phase: 'storing',
             filesProcessed: filesScanned,
             totalFiles: filesScanned,
             documentsIndexed,
+            totalDocuments: embeddingDocuments.length,
             percentComplete: 66 + (documentsIndexed / embeddingDocuments.length) * 33,
           });
         } catch (error) {
@@ -134,9 +160,15 @@ export class RepositoryIndexer {
             error: error instanceof Error ? error : undefined,
             timestamp: new Date(),
           });
+          logger?.error(
+            { batch: batchNum, error: error instanceof Error ? error.message : String(error) },
+            'Batch embedding failed'
+          );
         }
       }
 
+      logger?.info({ documentsIndexed, errors: errors.length }, 'Embedding complete');
+
       // Update state
       await this.updateState(scanResult.documents);
 
@@ -253,6 +285,7 @@ export class RepositoryIndexer {
         repoRoot: this.config.repositoryPath,
         include: filesToReindex,
         exclude: this.config.excludePatterns,
+        logger: options.logger,
       });
 
       documentsExtracted = scanResult.documents.length;
diff --git a/packages/core/src/indexer/types.ts b/packages/core/src/indexer/types.ts
@@ -2,6 +2,8 @@
  * Repository Indexer types
  */
 
+import type { Logger } from '@lytics/kero';
+
 /**
  * Options for indexing a repository
  */
@@ -20,6 +22,9 @@ export interface IndexOptions {
 
   /** Progress callback for tracking indexing */
   onProgress?: (progress: IndexProgress) => void;
+
+  /** Logger for progress and debug output */
+  logger?: Logger;
 }
 
 /**
@@ -46,6 +51,9 @@ export interface IndexProgress {
   /** Documents indexed so far */
   documentsIndexed: number;
 
+  /** Total documents to index (available during storing phase) */
+  totalDocuments?: number;
+
   /** Current file being processed */
   currentFile?: string;
 
diff --git a/packages/core/src/scanner/__tests__/scanner.test.ts b/packages/core/src/scanner/__tests__/scanner.test.ts
@@ -511,8 +511,8 @@ describe('Scanner', () => {
       const scannerInterface = result.documents.find((d) => d.metadata.name === 'Scanner');
       expect(scannerInterface).toBeDefined();
       expect(scannerInterface?.metadata.imports).toBeDefined();
-      // types.ts has no imports, so should be empty array
-      expect(scannerInterface?.metadata.imports).toEqual([]);
+      // types.ts imports Logger from @lytics/kero
+      expect(scannerInterface?.metadata.imports).toContain('@lytics/kero');
     });
 
     it('should extract imports for methods', async () => {
@@ -581,8 +581,8 @@ describe('Scanner', () => {
       expect(docs.length >= 0).toBe(true);
     });
 
-    it('should return empty array for files with no imports', async () => {
-      // types.ts should have no imports
+    it('should capture type-only imports', async () => {
+      // types.ts imports Logger type from @lytics/kero
       const result = await scanRepository({
         repoRoot,
         include: ['packages/core/src/scanner/types.ts'],
@@ -591,7 +591,8 @@ describe('Scanner', () => {
       const docType = result.documents.find((d) => d.metadata.name === 'DocumentType');
       expect(docType).toBeDefined();
       expect(docType?.metadata.imports).toBeDefined();
-      expect(docType?.metadata.imports).toEqual([]);
+      // Type imports should be captured
+      expect(docType?.metadata.imports).toContain('@lytics/kero');
     });
   });
 
diff --git a/packages/core/src/scanner/index.ts b/packages/core/src/scanner/index.ts
@@ -13,6 +13,7 @@ export type {
   Scanner,
   ScannerCapabilities,
   ScanOptions,
+  ScanProgress,
   ScanResult,
   ScanStats,
 } from './types';
diff --git a/packages/core/src/scanner/registry.ts b/packages/core/src/scanner/registry.ts
@@ -1,5 +1,5 @@
 import { globby } from 'globby';
-import type { Document, Scanner, ScanOptions, ScanResult } from './types';
+import type { Document, Scanner, ScanOptions, ScanProgress, ScanResult } from './types';
 
 /**
  * Scanner registry manages multiple language scanners
@@ -60,6 +60,24 @@ export class ScannerRegistry {
   async scanRepository(options: ScanOptions): Promise<ScanResult> {
     const startTime = Date.now();
     const errors: Array<{ file: string; error: string }> = [];
+    const logger = options.logger?.child({ component: 'scanner' });
+    const onProgress = options.onProgress;
+
+    // Helper to emit progress
+    const emitProgress = (progress: Partial<ScanProgress>) => {
+      onProgress?.({
+        phase: 'discovery',
+        filesTotal: 0,
+        filesScanned: 0,
+        documentsExtracted: 0,
+        errors: errors.length,
+        ...progress,
+      });
+    };
+
+    // Phase 1: Discovery
+    logger?.info({ repoRoot: options.repoRoot }, 'Starting repository scan');
+    emitProgress({ phase: 'discovery' });
 
     // Build glob patterns
     const patterns = this.buildGlobPatterns(options);
@@ -71,6 +89,8 @@ export class ScannerRegistry {
       absolute: false,
     });
 
+    logger?.info({ totalFiles: files.length }, 'File discovery complete');
+
     // Group files by scanner
     const filesByScanner = new Map<Scanner, string[]>();
 
@@ -83,23 +103,85 @@ export class ScannerRegistry {
       }
     }
 
-    // Scan files with appropriate scanners
+    // Log per-language breakdown
+    const languageBreakdown: Record<string, number> = {};
+    for (const [scanner, scannerFiles] of filesByScanner) {
+      languageBreakdown[scanner.language] = scannerFiles.length;
+      logger?.info(
+        { language: scanner.language, files: scannerFiles.length },
+        `Found ${scannerFiles.length} ${scanner.language} files`
+      );
+    }
+
+    // Phase 2: Scanning
     const allDocuments: Document[] = [];
+    let totalFilesScanned = 0;
 
     for (const [scanner, scannerFiles] of filesByScanner.entries()) {
+      logger?.debug(
+        { language: scanner.language, fileCount: scannerFiles.length },
+        `Scanning ${scanner.language}...`
+      );
+
+      emitProgress({
+        phase: 'scanning',
+        language: scanner.language,
+        filesTotal: files.length,
+        filesScanned: totalFilesScanned,
+        documentsExtracted: allDocuments.length,
+      });
+
       try {
         const documents = await scanner.scan(scannerFiles, options.repoRoot);
         allDocuments.push(...documents);
+        totalFilesScanned += scannerFiles.length;
+
+        logger?.info(
+          { language: scanner.language, files: scannerFiles.length, documents: documents.length },
+          `${scanner.language} scan complete`
+        );
+
+        emitProgress({
+          phase: 'scanning',
+          language: scanner.language,
+          filesTotal: files.length,
+          filesScanned: totalFilesScanned,
+          documentsExtracted: allDocuments.length,
+        });
       } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : String(error);
         errors.push({
           file: `[${scanner.language}]`,
-          error: error instanceof Error ? error.message : String(error),
+          error: errorMessage,
         });
+        logger?.error(
+          { language: scanner.language, error: errorMessage },
+          `${scanner.language} scan failed`
+        );
       }
     }
 
+    // Phase 3: Complete
     const duration = Date.now() - startTime;
 
+    logger?.info(
+      {
+        totalFiles: files.length,
+        totalDocuments: allDocuments.length,
+        duration: `${duration}ms`,
+        byLanguage: languageBreakdown,
+        errors: errors.length,
+      },
+      'Repository scan complete'
+    );
+
+    emitProgress({
+      phase: 'complete',
+      filesTotal: files.length,
+      filesScanned: totalFilesScanned,
+      documentsExtracted: allDocuments.length,
+    });
+
     return {
       documents: allDocuments,
       stats: {
@@ -152,6 +234,15 @@ export class ScannerRegistry {
       '**/.turbo/**',
       '**/.nuxt/**',
 
+      // Go generated files
+      '**/*.pb.go', // Protobuf
+      '**/*.gen.go', // Code generators
+      '**/*_gen.go', // Alternative generator pattern
+      '**/*.pb.gw.go', // gRPC gateway
+      '**/mock_*.go', // Mockgen files
+      '**/mocks/**', // Mock directories
+      '**/testdata/**', // Test fixtures
+
       // Version control
       '**/.git/**',
       '**/.svn/**',
diff --git a/packages/core/src/scanner/types.ts b/packages/core/src/scanner/types.ts
