mirrors/mcphub
1
0
Fork 0
mirror of https://github.com/samanhappy/mcphub.git synced 2025-12-31 20:00:00 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: expand environment variables in database and OpenAI configuration (#162)

Browse Source
This commit is contained in:
samanhappy
2025-06-02 19:46:39 +08:00
committed by GitHub
parent ecf79eaef3
commit c1621805de
5 changed files with 258 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

195
src/db/connection.ts
View File

@@ -1,13 +1,9 @@
import 'reflect-metadata'; // Ensure reflect-metadata is imported here too
import { DataSource, DataSourceOptions } from 'typeorm';
import dotenv from 'dotenv';
import dotenvExpand from 'dotenv-expand';
import path from 'path';
import { fileURLToPath } from 'url';
import entities from './entities/index.js';
import { registerPostgresVectorType } from './types/postgresVectorType.js';
import { VectorEmbeddingSubscriber } from './subscribers/VectorEmbeddingSubscriber.js';
import { loadSettings } from '../config/index.js';
import { getSmartRoutingConfig } from '../utils/smartRouting.js';
// Helper function to create required PostgreSQL extensions
const createRequiredExtensions = async (dataSource: DataSource): Promise<void> => {
@@ -30,23 +26,7 @@ const createRequiredExtensions = async (dataSource: DataSource): Promise<void> =
// Get database URL from smart routing config or fallback to environment variable
const getDatabaseUrl = (): string => {
try {
const settings = loadSettings();
const smartRouting = settings.systemConfig?.smartRouting;
// Use smart routing dbUrl if smart routing is enabled and dbUrl is configured
if (smartRouting?.enabled && smartRouting?.dbUrl) {
console.log('Using smart routing database URL');
return smartRouting.dbUrl;
}
} catch (error) {
console.warn(
'Failed to load settings for smart routing database URL, falling back to environment variable:',
error,
);
}
return '';
return getSmartRoutingConfig().dbUrl;
};
// Default database configuration
@@ -59,7 +39,10 @@ const defaultConfig: DataSourceOptions = {
};
// AppDataSource is the TypeORM data source
let AppDataSource = new DataSource(defaultConfig);
let appDataSource = new DataSource(defaultConfig);
// Global promise to track initialization status
let initializationPromise: Promise<DataSource> | null = null;
// Function to create a new DataSource with updated configuration
export const updateDataSourceConfig = (): DataSource => {
@@ -69,31 +52,36 @@ export const updateDataSourceConfig = (): DataSource => {
};
// If the configuration has changed, we need to create a new DataSource
const currentUrl = (AppDataSource.options as any).url;
const currentUrl = (appDataSource.options as any).url;
if (currentUrl !== newConfig.url) {
console.log('Database URL configuration changed, updating DataSource...');
AppDataSource = new DataSource(newConfig);
appDataSource = new DataSource(newConfig);
// Reset initialization promise when configuration changes
initializationPromise = null;
}
return AppDataSource;
return appDataSource;
};
// Get the current AppDataSource instance
export const getAppDataSource = (): DataSource => {
return AppDataSource;
return appDataSource;
};
// Reconnect database with updated configuration
export const reconnectDatabase = async (): Promise<DataSource> => {
try {
// Close existing connection if it exists
if (AppDataSource.isInitialized) {
if (appDataSource.isInitialized) {
console.log('Closing existing database connection...');
await AppDataSource.destroy();
await appDataSource.destroy();
}
// Reset initialization promise to allow fresh initialization
initializationPromise = null;
// Update configuration and reconnect
AppDataSource = updateDataSourceConfig();
appDataSource = updateDataSourceConfig();
return await initializeDatabase();
} catch (error) {
console.error('Error during database reconnection:', error);
@@ -101,26 +89,54 @@ export const reconnectDatabase = async (): Promise<DataSource> => {
}
};
// Initialize database connection
// Initialize database connection with concurrency control
export const initializeDatabase = async (): Promise<DataSource> => {
// If initialization is already in progress, wait for it to complete
if (initializationPromise) {
console.log('Database initialization already in progress, waiting for completion...');
return initializationPromise;
}
// If already initialized, return the existing instance
if (appDataSource.isInitialized) {
console.log('Database already initialized, returning existing instance');
return Promise.resolve(appDataSource);
}
// Create a new initialization promise
initializationPromise = performDatabaseInitialization();
try {
const result = await initializationPromise;
console.log('Database initialization completed successfully');
return result;
} catch (error) {
// Reset the promise on error so initialization can be retried
initializationPromise = null;
console.error('Database initialization failed:', error);
throw error;
}
};
// Internal function to perform the actual database initialization
const performDatabaseInitialization = async (): Promise<DataSource> => {
try {
// Update configuration before initializing
AppDataSource = updateDataSourceConfig();
appDataSource = updateDataSourceConfig();
if (!AppDataSource.isInitialized) {
if (!appDataSource.isInitialized) {
console.log('Initializing database connection...');
// Register the vector type with TypeORM
await AppDataSource.initialize();
registerPostgresVectorType(AppDataSource);
await appDataSource.initialize();
registerPostgresVectorType(appDataSource);
// Create required PostgreSQL extensions
await createRequiredExtensions(AppDataSource);
await createRequiredExtensions(appDataSource);
// Set up vector column and index with a more direct approach
try {
// Check if table exists first
const tableExists = await AppDataSource.query(`
const tableExists = await appDataSource.query(`
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'public'
@@ -134,7 +150,7 @@ export const initializeDatabase = async (): Promise<DataSource> => {
// Step 1: Drop any existing index on the column
try {
await AppDataSource.query(`DROP INDEX IF EXISTS idx_vector_embeddings_embedding;`);
await appDataSource.query(`DROP INDEX IF EXISTS idx_vector_embeddings_embedding;`);
} catch (dropError: any) {
console.warn('Note: Could not drop existing index:', dropError.message);
}
@@ -142,14 +158,14 @@ export const initializeDatabase = async (): Promise<DataSource> => {
// Step 2: Alter column type to vector (if it's not already)
try {
// Check column type first
const columnType = await AppDataSource.query(`
const columnType = await appDataSource.query(`
SELECT data_type FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = 'vector_embeddings'
AND column_name = 'embedding';
`);
if (columnType.length > 0 && columnType[0].data_type !== 'vector') {
await AppDataSource.query(`
await appDataSource.query(`
ALTER TABLE vector_embeddings
ALTER COLUMN embedding TYPE vector USING embedding::vector;
`);
@@ -163,7 +179,7 @@ export const initializeDatabase = async (): Promise<DataSource> => {
// Step 3: Try to create appropriate indices
try {
// First, let's check if there are any records to determine the dimensions
const records = await AppDataSource.query(`
const records = await appDataSource.query(`
SELECT dimensions FROM vector_embeddings LIMIT 1;
`);
@@ -177,13 +193,13 @@ export const initializeDatabase = async (): Promise<DataSource> => {
// Set the vector dimensions explicitly only if table has data
if (records && records.length > 0) {
await AppDataSource.query(`
await appDataSource.query(`
ALTER TABLE vector_embeddings
ALTER COLUMN embedding TYPE vector(${dimensions});
`);
// Now try to create the index
await AppDataSource.query(`
await appDataSource.query(`
CREATE INDEX IF NOT EXISTS idx_vector_embeddings_embedding
ON vector_embeddings USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
`);
@@ -199,7 +215,7 @@ export const initializeDatabase = async (): Promise<DataSource> => {
try {
// Try HNSW index instead
await AppDataSource.query(`
await appDataSource.query(`
CREATE INDEX IF NOT EXISTS idx_vector_embeddings_embedding
ON vector_embeddings USING hnsw (embedding vector_cosine_ops);
`);
@@ -210,7 +226,7 @@ export const initializeDatabase = async (): Promise<DataSource> => {
try {
// Create a basic GIN index as last resort
await AppDataSource.query(`
await appDataSource.query(`
CREATE INDEX IF NOT EXISTS idx_vector_embeddings_embedding
ON vector_embeddings USING gin (embedding);
`);
@@ -235,12 +251,11 @@ export const initializeDatabase = async (): Promise<DataSource> => {
// Run one final setup check after schema synchronization is done
if (defaultConfig.synchronize) {
setTimeout(async () => {
try {
console.log('Running final vector configuration check...');
try {
console.log('Running final vector configuration check...');
// Try setup again with the same code from above
const tableExists = await AppDataSource.query(`
// Try setup again with the same code from above
const tableExists = await appDataSource.query(`
SELECT EXISTS (
SELECT FROM information_schema.tables
WHERE table_schema = 'public'
@@ -248,64 +263,60 @@ export const initializeDatabase = async (): Promise<DataSource> => {
);
`);
if (tableExists[0].exists) {
console.log('Vector embeddings table found, checking configuration...');
if (tableExists[0].exists) {
console.log('Vector embeddings table found, checking configuration...');
// Get the dimension size first
try {
// Try to get dimensions from an existing record
const records = await AppDataSource.query(`
// Get the dimension size first
try {
// Try to get dimensions from an existing record
const records = await appDataSource.query(`
SELECT dimensions FROM vector_embeddings LIMIT 1;
`);
// Only proceed if we have existing data, otherwise let vector service handle it
if (records && records.length > 0 && records[0].dimensions) {
const dimensions = records[0].dimensions;
console.log(`Found vector dimension from database: ${dimensions}`);
// Only proceed if we have existing data, otherwise let vector service handle it
if (records && records.length > 0 && records[0].dimensions) {
const dimensions = records[0].dimensions;
console.log(`Found vector dimension from database: ${dimensions}`);
// Ensure column type is vector with explicit dimensions
await AppDataSource.query(`
// Ensure column type is vector with explicit dimensions
await appDataSource.query(`
ALTER TABLE vector_embeddings
ALTER COLUMN embedding TYPE vector(${dimensions});
`);
console.log('Vector embedding column type updated in final check.');
console.log('Vector embedding column type updated in final check.');
// One more attempt at creating the index with dimensions
try {
// Drop existing index if any
await AppDataSource.query(`
// One more attempt at creating the index with dimensions
try {
// Drop existing index if any
await appDataSource.query(`
DROP INDEX IF EXISTS idx_vector_embeddings_embedding;
`);
// Create new index with proper dimensions
await AppDataSource.query(`
// Create new index with proper dimensions
await appDataSource.query(`
CREATE INDEX idx_vector_embeddings_embedding
ON vector_embeddings USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
`);
console.log('Created IVFFlat index in final check.');
} catch (indexError: any) {
console.warn(
'Final index creation attempt did not succeed:',
indexError.message,
);
console.warn('Using basic lookup without vector index.');
}
} else {
console.log(
'No existing vector data found, vector dimensions will be configured by vector service.',
);
console.log('Created IVFFlat index in final check.');
} catch (indexError: any) {
console.warn('Final index creation attempt did not succeed:', indexError.message);
console.warn('Using basic lookup without vector index.');
}
} catch (setupError: any) {
console.warn('Vector setup in final check failed:', setupError.message);
} else {
console.log(
'No existing vector data found, vector dimensions will be configured by vector service.',
);
}
} catch (setupError: any) {
console.warn('Vector setup in final check failed:', setupError.message);
}
} catch (error: any) {
console.warn('Post-initialization vector setup failed:', error.message);
}
}, 3000); // Give synchronize some time to complete
} catch (error: any) {
console.warn('Post-initialization vector setup failed:', error.message);
}
}
}
return AppDataSource;
return appDataSource;
} catch (error) {
console.error('Error during database initialization:', error);
throw error;
@@ -314,18 +325,18 @@ export const initializeDatabase = async (): Promise<DataSource> => {
// Get database connection status
export const isDatabaseConnected = (): boolean => {
return AppDataSource.isInitialized;
return appDataSource.isInitialized;
};
// Close database connection
export const closeDatabase = async (): Promise<void> => {
if (AppDataSource.isInitialized) {
await AppDataSource.destroy();
if (appDataSource.isInitialized) {
await appDataSource.destroy();
console.log('Database connection closed.');
}
};
// Export AppDataSource for backward compatibility
export { AppDataSource };
export const AppDataSource = appDataSource;
export default getAppDataSource;