mirrors/mcphub
1
0
Fork 0
mirror of https://github.com/samanhappy/mcphub.git synced 2025-12-23 18:29:21 -05:00
Code Issues Packages Projects Releases Wiki Activity

Refactor smart routing configuration and async database handling (#519)

Browse Source
This commit is contained in:
samanhappy
2025-12-20 12:16:09 +08:00
committed by GitHub
parent eb1a965e45
commit 33eae50bd3
5 changed files with 122 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
frontend/src/pages/SettingsPage.tsx
View File

@@ -1233,24 +1233,27 @@ const SettingsPage: React.FC = () => {
/>
</div>
<div className="p-3 bg-gray-50 rounded-md">
<div className="mb-2">
<h3 className="font-medium text-gray-700">
<span className="text-red-500 px-1">*</span>
{t('settings.dbUrl')}
</h3>
{/* hide when DB_URL env is set */}
{smartRoutingConfig.dbUrl !== '${DB_URL}' && (
<div className="p-3 bg-gray-50 rounded-md">
<div className="mb-2">
<h3 className="font-medium text-gray-700">
<span className="text-red-500 px-1">*</span>
{t('settings.dbUrl')}
</h3>
</div>
<div className="flex items-center gap-3">
<input
type="text"
value={tempSmartRoutingConfig.dbUrl}
onChange={(e) => handleSmartRoutingConfigChange('dbUrl', e.target.value)}
placeholder={t('settings.dbUrlPlaceholder')}
className="flex-1 mt-1 block w-full py-2 px-3 border rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm border-gray-300 form-input"
disabled={loading}
/>
</div>
</div>
<div className="flex items-center gap-3">
<input
type="text"
value={tempSmartRoutingConfig.dbUrl}
onChange={(e) => handleSmartRoutingConfigChange('dbUrl', e.target.value)}
placeholder={t('settings.dbUrlPlaceholder')}
className="flex-1 mt-1 block w-full py-2 px-3 border rounded-md shadow-sm focus:outline-none focus:ring-blue-500 focus:border-blue-500 sm:text-sm border-gray-300 form-input"
disabled={loading}
/>
</div>
</div>
)}
<div className="p-3 bg-gray-50 rounded-md">
<div className="mb-2">

17
src/controllers/serverController.ts
View File

@@ -66,6 +66,20 @@ export const getAllSettings = async (_: Request, res: Response): Promise<void> =
const systemConfigDao = getSystemConfigDao();
const systemConfig = await systemConfigDao.get();
// Ensure smart routing config has DB URL set if environment variable is present
const dbUrlEnv = process.env.DB_URL || '';
if (!systemConfig.smartRouting) {
systemConfig.smartRouting = {
enabled: false,
dbUrl: dbUrlEnv ? '${DB_URL}' : '',
openaiApiBaseUrl: '',
openaiApiKey: '',
openaiApiEmbeddingModel: '',
};
} else if (!systemConfig.smartRouting.dbUrl) {
systemConfig.smartRouting.dbUrl = dbUrlEnv ? '${DB_URL}' : '';
}
// Get bearer auth keys from DAO
const bearerKeyDao = getBearerKeyDao();
const bearerKeys = await bearerKeyDao.findAll();
@@ -978,7 +992,8 @@ export const updateSystemConfig = async (req: Request, res: Response): Promise<v
if (typeof smartRouting.enabled === 'boolean') {
// If enabling Smart Routing, validate required fields
if (smartRouting.enabled) {
const currentDbUrl = smartRouting.dbUrl || systemConfig.smartRouting.dbUrl;
const currentDbUrl =
process.env.DB_URL || smartRouting.dbUrl || systemConfig.smartRouting.dbUrl;
const currentOpenaiApiKey =
smartRouting.openaiApiKey || systemConfig.smartRouting.openaiApiKey;

65
src/db/connection.ts
View File

@@ -25,39 +25,44 @@ const createRequiredExtensions = async (dataSource: DataSource): Promise<void> =
};
// Get database URL from smart routing config or fallback to environment variable
const getDatabaseUrl = (): string => {
return getSmartRoutingConfig().dbUrl;
const getDatabaseUrl = async (): Promise<string> => {
return (await getSmartRoutingConfig()).dbUrl;
};
// Default database configuration
const defaultConfig: DataSourceOptions = {
type: 'postgres',
url: getDatabaseUrl(),
synchronize: true,
entities: entities,
subscribers: [VectorEmbeddingSubscriber],
// Default database configuration (without URL - will be set during initialization)
const getDefaultConfig = async (): Promise<DataSourceOptions> => {
return {
type: 'postgres',
url: await getDatabaseUrl(),
synchronize: true,
entities: entities,
subscribers: [VectorEmbeddingSubscriber],
};
};
// AppDataSource is the TypeORM data source
let appDataSource = new DataSource(defaultConfig);
// AppDataSource is the TypeORM data source (initialized with empty config, will be updated)
let appDataSource: DataSource | null = null;
// Global promise to track initialization status
let initializationPromise: Promise<DataSource> | null = null;
// Function to create a new DataSource with updated configuration
export const updateDataSourceConfig = (): DataSource => {
const newConfig: DataSourceOptions = {
...defaultConfig,
url: getDatabaseUrl(),
};
export const updateDataSourceConfig = async (): Promise<DataSource> => {
const newConfig = await getDefaultConfig();
// If the configuration has changed, we need to create a new DataSource
const currentUrl = (appDataSource.options as any).url;
if (currentUrl !== newConfig.url) {
console.log('Database URL configuration changed, updating DataSource...');
if (appDataSource) {
const currentUrl = (appDataSource.options as any).url;
const newUrl = (newConfig as any).url;
if (currentUrl !== newUrl) {
console.log('Database URL configuration changed, updating DataSource...');
appDataSource = new DataSource(newConfig);
// Reset initialization promise when configuration changes
initializationPromise = null;
}
} else {
// First time initialization
appDataSource = new DataSource(newConfig);
// Reset initialization promise when configuration changes
initializationPromise = null;
}
return appDataSource;
@@ -65,6 +70,9 @@ export const updateDataSourceConfig = (): DataSource => {
// Get the current AppDataSource instance
export const getAppDataSource = (): DataSource => {
if (!appDataSource) {
throw new Error('Database not initialized. Call initializeDatabase() first.');
}
return appDataSource;
};
@@ -72,7 +80,7 @@ export const getAppDataSource = (): DataSource => {
export const reconnectDatabase = async (): Promise<DataSource> => {
try {
// Close existing connection if it exists
if (appDataSource.isInitialized) {
if (appDataSource && appDataSource.isInitialized) {
console.log('Closing existing database connection...');
await appDataSource.destroy();
}
@@ -81,7 +89,7 @@ export const reconnectDatabase = async (): Promise<DataSource> => {
initializationPromise = null;
// Update configuration and reconnect
appDataSource = updateDataSourceConfig();
appDataSource = await updateDataSourceConfig();
return await initializeDatabase();
} catch (error) {
console.error('Error during database reconnection:', error);
@@ -98,7 +106,7 @@ export const initializeDatabase = async (): Promise<DataSource> => {
}
// If already initialized, return the existing instance
if (appDataSource.isInitialized) {
if (appDataSource && appDataSource.isInitialized) {
console.log('Database already initialized, returning existing instance');
return Promise.resolve(appDataSource);
}
@@ -122,7 +130,7 @@ export const initializeDatabase = async (): Promise<DataSource> => {
const performDatabaseInitialization = async (): Promise<DataSource> => {
try {
// Update configuration before initializing
appDataSource = updateDataSourceConfig();
appDataSource = await updateDataSourceConfig();
if (!appDataSource.isInitialized) {
console.log('Initializing database connection...');
@@ -250,7 +258,8 @@ const performDatabaseInitialization = async (): Promise<DataSource> => {
console.log('Database connection established successfully.');
// Run one final setup check after schema synchronization is done
if (defaultConfig.synchronize) {
const config = await getDefaultConfig();
if (config.synchronize) {
try {
console.log('Running final vector configuration check...');
@@ -325,12 +334,12 @@ const performDatabaseInitialization = async (): Promise<DataSource> => {
// Get database connection status
export const isDatabaseConnected = (): boolean => {
return appDataSource.isInitialized;
return appDataSource ? appDataSource.isInitialized : false;
};
// Close database connection
export const closeDatabase = async (): Promise<void> => {
if (appDataSource.isInitialized) {
if (appDataSource && appDataSource.isInitialized) {
await appDataSource.destroy();
console.log('Database connection closed.');
}

95
src/services/vectorSearchService.ts
View File

@@ -6,8 +6,8 @@ import { getSmartRoutingConfig } from '../utils/smartRouting.js';
import OpenAI from 'openai';
// Get OpenAI configuration from smartRouting settings or fallback to environment variables
const getOpenAIConfig = () => {
const smartRoutingConfig = getSmartRoutingConfig();
const getOpenAIConfig = async () => {
const smartRoutingConfig = await getSmartRoutingConfig();
return {
apiKey: smartRoutingConfig.openaiApiKey,
baseURL: smartRoutingConfig.openaiApiBaseUrl,
@@ -34,8 +34,8 @@ const getDimensionsForModel = (model: string): number => {
};
// Initialize the OpenAI client with smartRouting configuration
const getOpenAIClient = () => {
const config = getOpenAIConfig();
const getOpenAIClient = async () => {
const config = await getOpenAIConfig();
return new OpenAI({
apiKey: config.apiKey, // Get API key from smartRouting settings or environment variables
baseURL: config.baseURL, // Get base URL from smartRouting settings or fallback to default
@@ -53,32 +53,26 @@ const getOpenAIClient = () => {
* @returns Promise with vector embedding as number array
*/
async function generateEmbedding(text: string): Promise<number[]> {
try {
const config = getOpenAIConfig();
const openai = getOpenAIClient();
const config = await getOpenAIConfig();
const openai = await getOpenAIClient();
// Check if API key is configured
if (!openai.apiKey) {
console.warn('OpenAI API key is not configured. Using fallback embedding method.');
return generateFallbackEmbedding(text);
}
// Truncate text if it's too long (OpenAI has token limits)
const truncatedText = text.length > 8000 ? text.substring(0, 8000) : text;
// Call OpenAI's embeddings API
const response = await openai.embeddings.create({
model: config.embeddingModel, // Modern model with better performance
input: truncatedText,
});
// Return the embedding
return response.data[0].embedding;
} catch (error) {
console.error('Error generating embedding:', error);
console.warn('Falling back to simple embedding method');
// Check if API key is configured
if (!openai.apiKey) {
console.warn('OpenAI API key is not configured. Using fallback embedding method.');
return generateFallbackEmbedding(text);
}
// Truncate text if it's too long (OpenAI has token limits)
const truncatedText = text.length > 8000 ? text.substring(0, 8000) : text;
// Call OpenAI's embeddings API
const response = await openai.embeddings.create({
model: config.embeddingModel, // Modern model with better performance
input: truncatedText,
});
// Return the embedding
return response.data[0].embedding;
}
/**
@@ -198,12 +192,12 @@ export const saveToolsAsVectorEmbeddings = async (
return;
}
const smartRoutingConfig = getSmartRoutingConfig();
const smartRoutingConfig = await getSmartRoutingConfig();
if (!smartRoutingConfig.enabled) {
return;
}
const config = getOpenAIConfig();
const config = await getOpenAIConfig();
const vectorRepository = getRepositoryFactory(
'vectorEmbeddings',
)() as VectorEmbeddingRepository;
@@ -227,31 +221,26 @@ export const saveToolsAsVectorEmbeddings = async (
.filter(Boolean)
.join(' ');
try {
// Generate embedding
const embedding = await generateEmbedding(searchableText);
// Generate embedding
const embedding = await generateEmbedding(searchableText);
// Check database compatibility before saving
await checkDatabaseVectorDimensions(embedding.length);
// Check database compatibility before saving
await checkDatabaseVectorDimensions(embedding.length);
// Save embedding
await vectorRepository.saveEmbedding(
'tool',
`${serverName}:${tool.name}`,
searchableText,
embedding,
{
serverName,
toolName: tool.name,
description: tool.description,
inputSchema: tool.inputSchema,
},
config.embeddingModel, // Store the model used for this embedding
);
} catch (toolError) {
console.error(`Error processing tool ${tool.name} for server ${serverName}:`, toolError);
// Continue with the next tool rather than failing the whole batch
}
// Save embedding
await vectorRepository.saveEmbedding(
'tool',
`${serverName}:${tool.name}`,
searchableText,
embedding,
{
serverName,
toolName: tool.name,
description: tool.description,
inputSchema: tool.inputSchema,
},
config.embeddingModel, // Store the model used for this embedding
);
}
console.log(`Saved ${tools.length} tool embeddings for server: ${serverName}`);
@@ -381,7 +370,7 @@ export const getAllVectorizedTools = async (
}>
> => {
try {
const config = getOpenAIConfig();
const config = await getOpenAIConfig();
const vectorRepository = getRepositoryFactory(
'vectorEmbeddings',
)() as VectorEmbeddingRepository;

12
src/utils/smartRouting.ts
View File

@@ -1,4 +1,5 @@
import { loadSettings, expandEnvVars } from '../config/index.js';
import { expandEnvVars } from '../config/index.js';
import { getSystemConfigDao } from '../dao/DaoFactory.js';
/**
* Smart routing configuration interface
@@ -22,10 +23,11 @@ export interface SmartRoutingConfig {
*
* @returns {SmartRoutingConfig} Complete smart routing configuration
*/
export function getSmartRoutingConfig(): SmartRoutingConfig {
const settings = loadSettings();
const smartRoutingSettings: Partial<SmartRoutingConfig> =
settings.systemConfig?.smartRouting || {};
export async function getSmartRoutingConfig(): Promise<SmartRoutingConfig> {
// Get system config from DAO
const systemConfigDao = getSystemConfigDao();
const systemConfig = await systemConfigDao.get();
const smartRoutingSettings: Partial<SmartRoutingConfig> = systemConfig.smartRouting || {};
return {
// Enabled status - check multiple environment variables