mirrors/mcphub
1
0
Fork 0
mirror of https://github.com/samanhappy/mcphub.git synced 2025-12-23 18:29:21 -05:00
Code Issues Packages Projects Releases Wiki Activity

Refactor smart routing configuration and async database handling (#519)

Browse Source
This commit is contained in:
samanhappy
2025-12-20 12:16:09 +08:00
committed by GitHub
parent eb1a965e45
commit 33eae50bd3
5 changed files with 122 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
frontend/src/pages/SettingsPage.tsx
View File

@@ -1233,24 +1233,27 @@ const SettingsPage: React.FC = () => {
/> />
</div> </div>
<div className="p-3 bg-gray-50 rounded-md"> {/* hide when DB_URL env is set */}
<div className="mb-2"> {smartRoutingConfig.dbUrl !== '${DB_URL}' && (
<h3 className="font-medium text-gray-700"> <div className="p-3 bg-gray-50 rounded-md">
<span className="text-red-500 px-1">*</span> <div className="mb-2">
{t('settings.dbUrl')} <h3 className="font-medium text-gray-700">
</h3> <span className="text-red-500 px-1">*</span>
{t('settings.dbUrl')}
</h3>
</div>
<div className="flex items-center gap-3">
<input
type="text"
value={tempSmartRoutingConfig.dbUrl}
onChange={(e) => handleSmartRoutingConfigChange('dbUrl', e.target.value)}
placeholder={t('settings.dbUrlPlaceholder')}
className="flex-1 mt-1 block w-full py-2 px-3 border rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm border-gray-300 form-input"
disabled={loading}
/>
</div>
</div> </div>
<div className="flex items-center gap-3"> )}
<input
type="text"
value={tempSmartRoutingConfig.dbUrl}
onChange={(e) => handleSmartRoutingConfigChange('dbUrl', e.target.value)}
placeholder={t('settings.dbUrlPlaceholder')}
className="flex-1 mt-1 block w-full py-2 px-3 border rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm border-gray-300 form-input"
disabled={loading}
/>
</div>
</div>
<div className="p-3 bg-gray-50 rounded-md"> <div className="p-3 bg-gray-50 rounded-md">
<div className="mb-2"> <div className="mb-2">

17
src/controllers/serverController.ts
View File

@@ -66,6 +66,20 @@ export const getAllSettings = async (_: Request, res: Response): Promise<void> =
const systemConfigDao = getSystemConfigDao(); const systemConfigDao = getSystemConfigDao();
const systemConfig = await systemConfigDao.get(); const systemConfig = await systemConfigDao.get();
// Ensure smart routing config has DB URL set if environment variable is present
const dbUrlEnv = process.env.DB_URL || '';
if (!systemConfig.smartRouting) {
systemConfig.smartRouting = {
enabled: false,
dbUrl: dbUrlEnv ? '${DB_URL}' : '',
openaiApiBaseUrl: '',
openaiApiKey: '',
openaiApiEmbeddingModel: '',
};
} else if (!systemConfig.smartRouting.dbUrl) {
systemConfig.smartRouting.dbUrl = dbUrlEnv ? '${DB_URL}' : '';
}
// Get bearer auth keys from DAO // Get bearer auth keys from DAO
const bearerKeyDao = getBearerKeyDao(); const bearerKeyDao = getBearerKeyDao();
const bearerKeys = await bearerKeyDao.findAll(); const bearerKeys = await bearerKeyDao.findAll();
@@ -978,7 +992,8 @@ export const updateSystemConfig = async (req: Request, res: Response): Promise<v
if (typeof smartRouting.enabled === 'boolean') { if (typeof smartRouting.enabled === 'boolean') {
// If enabling Smart Routing, validate required fields // If enabling Smart Routing, validate required fields
if (smartRouting.enabled) { if (smartRouting.enabled) {
const currentDbUrl = smartRouting.dbUrl || systemConfig.smartRouting.dbUrl; const currentDbUrl =
process.env.DB_URL || smartRouting.dbUrl || systemConfig.smartRouting.dbUrl;
const currentOpenaiApiKey = const currentOpenaiApiKey =
smartRouting.openaiApiKey || systemConfig.smartRouting.openaiApiKey; smartRouting.openaiApiKey || systemConfig.smartRouting.openaiApiKey;

65
src/db/connection.ts
View File

@@ -25,39 +25,44 @@ const createRequiredExtensions = async (dataSource: DataSource): Promise<void> =
}; };
// Get database URL from smart routing config or fallback to environment variable // Get database URL from smart routing config or fallback to environment variable
const getDatabaseUrl = (): string => { const getDatabaseUrl = async (): Promise<string> => {
return getSmartRoutingConfig().dbUrl; return (await getSmartRoutingConfig()).dbUrl;
}; };
// Default database configuration // Default database configuration (without URL - will be set during initialization)
const defaultConfig: DataSourceOptions = { const getDefaultConfig = async (): Promise<DataSourceOptions> => {
type: 'postgres', return {
url: getDatabaseUrl(), type: 'postgres',
synchronize: true, url: await getDatabaseUrl(),
entities: entities, synchronize: true,
subscribers: [VectorEmbeddingSubscriber], entities: entities,
subscribers: [VectorEmbeddingSubscriber],
};
}; };
// AppDataSource is the TypeORM data source // AppDataSource is the TypeORM data source (initialized with empty config, will be updated)
let appDataSource = new DataSource(defaultConfig); let appDataSource: DataSource | null = null;
// Global promise to track initialization status // Global promise to track initialization status
let initializationPromise: Promise<DataSource> | null = null; let initializationPromise: Promise<DataSource> | null = null;
// Function to create a new DataSource with updated configuration // Function to create a new DataSource with updated configuration
export const updateDataSourceConfig = (): DataSource => { export const updateDataSourceConfig = async (): Promise<DataSource> => {
const newConfig: DataSourceOptions = { const newConfig = await getDefaultConfig();
...defaultConfig,
url: getDatabaseUrl(),
};
// If the configuration has changed, we need to create a new DataSource // If the configuration has changed, we need to create a new DataSource
const currentUrl = (appDataSource.options as any).url; if (appDataSource) {
if (currentUrl !== newConfig.url) { const currentUrl = (appDataSource.options as any).url;
console.log('Database URL configuration changed, updating DataSource...'); const newUrl = (newConfig as any).url;
if (currentUrl !== newUrl) {
console.log('Database URL configuration changed, updating DataSource...');
appDataSource = new DataSource(newConfig);
// Reset initialization promise when configuration changes
initializationPromise = null;
}
} else {
// First time initialization
appDataSource = new DataSource(newConfig); appDataSource = new DataSource(newConfig);
// Reset initialization promise when configuration changes
initializationPromise = null;
} }
return appDataSource; return appDataSource;
@@ -65,6 +70,9 @@ export const updateDataSourceConfig = (): DataSource => {
// Get the current AppDataSource instance // Get the current AppDataSource instance
export const getAppDataSource = (): DataSource => { export const getAppDataSource = (): DataSource => {
if (!appDataSource) {
throw new Error('Database not initialized. Call initializeDatabase() first.');
}
return appDataSource; return appDataSource;
}; };
@@ -72,7 +80,7 @@ export const getAppDataSource = (): DataSource => {
export const reconnectDatabase = async (): Promise<DataSource> => { export const reconnectDatabase = async (): Promise<DataSource> => {
try { try {
// Close existing connection if it exists // Close existing connection if it exists
if (appDataSource.isInitialized) { if (appDataSource && appDataSource.isInitialized) {
console.log('Closing existing database connection...'); console.log('Closing existing database connection...');
await appDataSource.destroy(); await appDataSource.destroy();
} }
@@ -81,7 +89,7 @@ export const reconnectDatabase = async (): Promise<DataSource> => {
initializationPromise = null; initializationPromise = null;
// Update configuration and reconnect // Update configuration and reconnect
appDataSource = updateDataSourceConfig(); appDataSource = await updateDataSourceConfig();
return await initializeDatabase(); return await initializeDatabase();
} catch (error) { } catch (error) {
console.error('Error during database reconnection:', error); console.error('Error during database reconnection:', error);
@@ -98,7 +106,7 @@ export const initializeDatabase = async (): Promise<DataSource> => {
} }
// If already initialized, return the existing instance // If already initialized, return the existing instance
if (appDataSource.isInitialized) { if (appDataSource && appDataSource.isInitialized) {
console.log('Database already initialized, returning existing instance'); console.log('Database already initialized, returning existing instance');
return Promise.resolve(appDataSource); return Promise.resolve(appDataSource);
} }
@@ -122,7 +130,7 @@ export const initializeDatabase = async (): Promise<DataSource> => {
const performDatabaseInitialization = async (): Promise<DataSource> => { const performDatabaseInitialization = async (): Promise<DataSource> => {
try { try {
// Update configuration before initializing // Update configuration before initializing
appDataSource = updateDataSourceConfig(); appDataSource = await updateDataSourceConfig();
if (!appDataSource.isInitialized) { if (!appDataSource.isInitialized) {
console.log('Initializing database connection...'); console.log('Initializing database connection...');
@@ -250,7 +258,8 @@ const performDatabaseInitialization = async (): Promise<DataSource> => {
console.log('Database connection established successfully.'); console.log('Database connection established successfully.');
// Run one final setup check after schema synchronization is done // Run one final setup check after schema synchronization is done
if (defaultConfig.synchronize) { const config = await getDefaultConfig();
if (config.synchronize) {
try { try {
console.log('Running final vector configuration check...'); console.log('Running final vector configuration check...');
@@ -325,12 +334,12 @@ const performDatabaseInitialization = async (): Promise<DataSource> => {
// Get database connection status // Get database connection status
export const isDatabaseConnected = (): boolean => { export const isDatabaseConnected = (): boolean => {
return appDataSource.isInitialized; return appDataSource ? appDataSource.isInitialized : false;
}; };
// Close database connection // Close database connection
export const closeDatabase = async (): Promise<void> => { export const closeDatabase = async (): Promise<void> => {
if (appDataSource.isInitialized) { if (appDataSource && appDataSource.isInitialized) {
await appDataSource.destroy(); await appDataSource.destroy();
console.log('Database connection closed.'); console.log('Database connection closed.');
} }

95
src/services/vectorSearchService.ts
View File

@@ -6,8 +6,8 @@ import { getSmartRoutingConfig } from '../utils/smartRouting.js';
import OpenAI from 'openai'; import OpenAI from 'openai';
// Get OpenAI configuration from smartRouting settings or fallback to environment variables // Get OpenAI configuration from smartRouting settings or fallback to environment variables
const getOpenAIConfig = () => { const getOpenAIConfig = async () => {
const smartRoutingConfig = getSmartRoutingConfig(); const smartRoutingConfig = await getSmartRoutingConfig();
return { return {
apiKey: smartRoutingConfig.openaiApiKey, apiKey: smartRoutingConfig.openaiApiKey,
baseURL: smartRoutingConfig.openaiApiBaseUrl, baseURL: smartRoutingConfig.openaiApiBaseUrl,
@@ -34,8 +34,8 @@ const getDimensionsForModel = (model: string): number => {
}; };
// Initialize the OpenAI client with smartRouting configuration // Initialize the OpenAI client with smartRouting configuration
const getOpenAIClient = () => { const getOpenAIClient = async () => {
const config = getOpenAIConfig(); const config = await getOpenAIConfig();
return new OpenAI({ return new OpenAI({
apiKey: config.apiKey, // Get API key from smartRouting settings or environment variables apiKey: config.apiKey, // Get API key from smartRouting settings or environment variables
baseURL: config.baseURL, // Get base URL from smartRouting settings or fallback to default baseURL: config.baseURL, // Get base URL from smartRouting settings or fallback to default
@@ -53,32 +53,26 @@ const getOpenAIClient = () => {
* @returns Promise with vector embedding as number array * @returns Promise with vector embedding as number array
*/ */
async function generateEmbedding(text: string): Promise<number[]> { async function generateEmbedding(text: string): Promise<number[]> {
try { const config = await getOpenAIConfig();
const config = getOpenAIConfig(); const openai = await getOpenAIClient();
const openai = getOpenAIClient();
// Check if API key is configured // Check if API key is configured
if (!openai.apiKey) { if (!openai.apiKey) {
console.warn('OpenAI API key is not configured. Using fallback embedding method.'); console.warn('OpenAI API key is not configured. Using fallback embedding method.');
return generateFallbackEmbedding(text);
}
// Truncate text if it's too long (OpenAI has token limits)
const truncatedText = text.length > 8000 ? text.substring(0, 8000) : text;
// Call OpenAI's embeddings API
const response = await openai.embeddings.create({
model: config.embeddingModel, // Modern model with better performance
input: truncatedText,
});
// Return the embedding
return response.data[0].embedding;
} catch (error) {
console.error('Error generating embedding:', error);
console.warn('Falling back to simple embedding method');
return generateFallbackEmbedding(text); return generateFallbackEmbedding(text);
} }
// Truncate text if it's too long (OpenAI has token limits)
const truncatedText = text.length > 8000 ? text.substring(0, 8000) : text;
// Call OpenAI's embeddings API
const response = await openai.embeddings.create({
model: config.embeddingModel, // Modern model with better performance
input: truncatedText,
});
// Return the embedding
return response.data[0].embedding;
} }
/** /**
@@ -198,12 +192,12 @@ export const saveToolsAsVectorEmbeddings = async (
return; return;
} }
const smartRoutingConfig = getSmartRoutingConfig(); const smartRoutingConfig = await getSmartRoutingConfig();
if (!smartRoutingConfig.enabled) { if (!smartRoutingConfig.enabled) {
return; return;
} }
const config = getOpenAIConfig(); const config = await getOpenAIConfig();
const vectorRepository = getRepositoryFactory( const vectorRepository = getRepositoryFactory(
'vectorEmbeddings', 'vectorEmbeddings',
)() as VectorEmbeddingRepository; )() as VectorEmbeddingRepository;
@@ -227,31 +221,26 @@ export const saveToolsAsVectorEmbeddings = async (
.filter(Boolean) .filter(Boolean)
.join(' '); .join(' ');
try { // Generate embedding
// Generate embedding const embedding = await generateEmbedding(searchableText);
const embedding = await generateEmbedding(searchableText);
// Check database compatibility before saving // Check database compatibility before saving
await checkDatabaseVectorDimensions(embedding.length); await checkDatabaseVectorDimensions(embedding.length);
// Save embedding // Save embedding
await vectorRepository.saveEmbedding( await vectorRepository.saveEmbedding(
'tool', 'tool',
`${serverName}:${tool.name}`, `${serverName}:${tool.name}`,
searchableText, searchableText,
embedding, embedding,
{ {
serverName, serverName,
toolName: tool.name, toolName: tool.name,
description: tool.description, description: tool.description,
inputSchema: tool.inputSchema, inputSchema: tool.inputSchema,
}, },
config.embeddingModel, // Store the model used for this embedding config.embeddingModel, // Store the model used for this embedding
); );
} catch (toolError) {
console.error(`Error processing tool ${tool.name} for server ${serverName}:`, toolError);
// Continue with the next tool rather than failing the whole batch
}
} }
console.log(`Saved ${tools.length} tool embeddings for server: ${serverName}`); console.log(`Saved ${tools.length} tool embeddings for server: ${serverName}`);
@@ -381,7 +370,7 @@ export const getAllVectorizedTools = async (
}> }>
> => { > => {
try { try {
const config = getOpenAIConfig(); const config = await getOpenAIConfig();
const vectorRepository = getRepositoryFactory( const vectorRepository = getRepositoryFactory(
'vectorEmbeddings', 'vectorEmbeddings',
)() as VectorEmbeddingRepository; )() as VectorEmbeddingRepository;

12
src/utils/smartRouting.ts
View File

@@ -1,4 +1,5 @@
import { loadSettings, expandEnvVars } from '../config/index.js'; import { expandEnvVars } from '../config/index.js';
import { getSystemConfigDao } from '../dao/DaoFactory.js';
/** /**
* Smart routing configuration interface * Smart routing configuration interface
@@ -22,10 +23,11 @@ export interface SmartRoutingConfig {
* *
* @returns {SmartRoutingConfig} Complete smart routing configuration * @returns {SmartRoutingConfig} Complete smart routing configuration
*/ */
export function getSmartRoutingConfig(): SmartRoutingConfig { export async function getSmartRoutingConfig(): Promise<SmartRoutingConfig> {
const settings = loadSettings(); // Get system config from DAO
const smartRoutingSettings: Partial<SmartRoutingConfig> = const systemConfigDao = getSystemConfigDao();
settings.systemConfig?.smartRouting || {}; const systemConfig = await systemConfigDao.get();
const smartRoutingSettings: Partial<SmartRoutingConfig> = systemConfig.smartRouting || {};
return { return {
// Enabled status - check multiple environment variables // Enabled status - check multiple environment variables