cim_summary/backend/test-vector-database.js

const { Pool } = require('pg');

// Load environment variables
require('dotenv').config();

const config = {
  database: {
    url: process.env.DATABASE_URL || 'postgresql://postgres:password@localhost:5432/cim_processor'
  }
};

async function testVectorDatabase() {
  console.log('🧪 Testing Vector Database Setup...\n');

  const pool = new Pool({
    connectionString: config.database.url
  });

  try {
    // Test 1: Check if pgvector extension is available
    console.log('1. Testing pgvector extension...');
    const extensionResult = await pool.query(`
      SELECT extname, extversion 
      FROM pg_extension 
      WHERE extname = 'vector'
    `);
    
    if (extensionResult.rows.length > 0) {
      console.log('✅ pgvector extension is installed and active');
      console.log(`   Version: ${extensionResult.rows[0].extversion}\n`);
    } else {
      console.log('❌ pgvector extension is not installed\n');
      return;
    }

    // Test 2: Check if vector tables exist
    console.log('2. Testing vector database tables...');
    const tablesResult = await pool.query(`
      SELECT table_name 
      FROM information_schema.tables 
      WHERE table_schema = 'public' 
      AND table_name IN ('document_chunks', 'vector_similarity_searches', 'document_similarities', 'industry_embeddings')
      ORDER BY table_name
    `);

    const expectedTables = ['document_chunks', 'vector_similarity_searches', 'document_similarities', 'industry_embeddings'];
    const foundTables = tablesResult.rows.map(row => row.table_name);

    console.log('   Expected tables:', expectedTables);
    console.log('   Found tables:', foundTables);

    if (foundTables.length === expectedTables.length) {
      console.log('✅ All vector database tables exist\n');
    } else {
      console.log('❌ Some vector database tables are missing\n');
      return;
    }

    // Test 3: Test vector column type
    console.log('3. Testing vector column type...');
    const vectorColumnResult = await pool.query(`
      SELECT column_name, data_type 
      FROM information_schema.columns 
      WHERE table_name = 'document_chunks' 
      AND column_name = 'embedding'
    `);

    if (vectorColumnResult.rows.length > 0 && vectorColumnResult.rows[0].data_type === 'USER-DEFINED') {
      console.log('✅ Vector column type is properly configured\n');
    } else {
      console.log('❌ Vector column type is not properly configured\n');
      return;
    }

    // Test 4: Test vector similarity function
    console.log('4. Testing vector similarity functions...');
    const functionResult = await pool.query(`
      SELECT routine_name 
      FROM information_schema.routines 
      WHERE routine_name IN ('cosine_similarity', 'find_similar_documents', 'update_document_similarities')
      ORDER BY routine_name
    `);

    const expectedFunctions = ['cosine_similarity', 'find_similar_documents', 'update_document_similarities'];
    const foundFunctions = functionResult.rows.map(row => row.routine_name);

    console.log('   Expected functions:', expectedFunctions);
    console.log('   Found functions:', foundFunctions);

    if (foundFunctions.length === expectedFunctions.length) {
      console.log('✅ All vector similarity functions exist\n');
    } else {
      console.log('❌ Some vector similarity functions are missing\n');
      return;
    }

    // Test 5: Test vector operations with sample data
    console.log('5. Testing vector operations with sample data...');
    
    // Create a sample vector (1536 dimensions for OpenAI text-embedding-3-small)
    // pgvector expects a string representation like '[1,2,3]'
    const sampleVector = '[' + Array.from({ length: 1536 }, () => Math.random().toFixed(6)).join(',') + ']';
    
    // Insert a test document chunk
    const { v4: uuidv4 } = require('uuid');
    const testDocumentId = uuidv4();
    const testChunkId = uuidv4();
    
    // First create a test document
    await pool.query(`
      INSERT INTO documents (
        id, original_file_name, file_path, file_size, status, user_id
      ) VALUES (
        $1, $2, $3, $4, $5, $6
      )
    `, [
      testDocumentId,
      'test-document.pdf',
      '/test/path',
      1024,
      'completed',
      'ea01b025-15e4-471e-8b54-c9ec519aa9ed' // Use an existing user ID
    ]);
    
    // Then insert the document chunk
    await pool.query(`
      INSERT INTO document_chunks (
        id, document_id, content, metadata, embedding, chunk_index, section
      ) VALUES (
        $1, $2, $3, $4, $5, $6, $7
      )
    `, [
      testChunkId,
      testDocumentId,
      'This is a test document chunk for vector database testing.',
      JSON.stringify({ test: true, timestamp: new Date().toISOString() }),
      sampleVector,
      0,
      'test_section'
    ]);

    console.log('   ✅ Inserted test document chunk');

    // Test vector similarity search
    const searchResult = await pool.query(`
      SELECT 
        document_id,
        content,
        1 - (embedding <=> $1) as similarity_score
      FROM document_chunks 
      WHERE embedding IS NOT NULL
      ORDER BY embedding <=> $1
      LIMIT 5
    `, [sampleVector]);

    if (searchResult.rows.length > 0) {
      console.log('   ✅ Vector similarity search works');
      console.log(`   Found ${searchResult.rows.length} results`);
      console.log(`   Top similarity score: ${searchResult.rows[0].similarity_score.toFixed(4)}`);
    } else {
      console.log('   ❌ Vector similarity search failed');
    }

    // Test cosine similarity function
    const cosineResult = await pool.query(`
      SELECT cosine_similarity($1, $1) as self_similarity
    `, [sampleVector]);

    if (cosineResult.rows.length > 0) {
      const selfSimilarity = parseFloat(cosineResult.rows[0].self_similarity);
      console.log(`   ✅ Cosine similarity function works (self-similarity: ${selfSimilarity.toFixed(4)})`);
    } else {
      console.log('   ❌ Cosine similarity function failed');
    }

    // Clean up test data
    await pool.query('DELETE FROM document_chunks WHERE document_id = $1', [testDocumentId]);
    await pool.query('DELETE FROM documents WHERE id = $1', [testDocumentId]);
    console.log('   ✅ Cleaned up test data\n');

    // Test 6: Check vector indexes
    console.log('6. Testing vector indexes...');
    const indexResult = await pool.query(`
      SELECT indexname, indexdef 
      FROM pg_indexes 
      WHERE tablename = 'document_chunks' 
      AND indexdef LIKE '%vector%'
    `);

    if (indexResult.rows.length > 0) {
      console.log('✅ Vector indexes exist:');
      indexResult.rows.forEach(row => {
        console.log(`   - ${row.indexname}`);
      });
    } else {
      console.log('❌ Vector indexes are missing');
    }

    console.log('\n🎉 Vector Database Test Completed Successfully!');
    console.log('\n📊 Summary:');
    console.log('   ✅ pgvector extension is active');
    console.log('   ✅ All required tables exist');
    console.log('   ✅ Vector column type is configured');
    console.log('   ✅ Vector similarity functions work');
    console.log('   ✅ Vector operations are functional');
    console.log('   ✅ Vector indexes are in place');
    
    console.log('\n🚀 Your vector database is ready for CIM processing!');

  } catch (error) {
    console.error('❌ Vector database test failed:', error.message);
    console.error('Stack trace:', error.stack);
  } finally {
    await pool.end();
  }
}

// Run the test
testVectorDatabase().catch(console.error);