b40f5b788c
- Delete stub scriptDownloader.js that contained placeholder implementation - Implement real JavaScript script downloader with GitHub fetch functionality - Fix incremental JSON sync to only process newly synced files - Add proper error handling and file structure management - Support all script types (ct/, tools/, vm/, vw/) with directory preservation - Download install scripts for CT scripts - Re-enable auto-sync service to use real implementation Scripts now download real content from GitHub instead of placeholders.
632 lines
21 KiB
JavaScript
632 lines
21 KiB
JavaScript
import cron from 'node-cron';
|
|
import { githubJsonService } from './githubJsonService.js';
|
|
import { scriptDownloaderService } from './scriptDownloader.js';
|
|
import { appriseService } from './appriseService.js';
|
|
import { readFile, writeFile, readFileSync, writeFileSync } from 'fs';
|
|
import { join } from 'path';
|
|
import cronValidator from 'cron-validator';
|
|
|
|
export class AutoSyncService {
|
|
constructor() {
|
|
this.cronJob = null;
|
|
this.isRunning = false;
|
|
}
|
|
|
|
/**
|
|
* Safely convert a date to ISO string, handling invalid dates
|
|
* @param {Date} date - The date to convert
|
|
* @returns {string} - ISO string or fallback timestamp
|
|
*/
|
|
safeToISOString(date) {
|
|
try {
|
|
// Check if the date is valid
|
|
if (!date || isNaN(date.getTime())) {
|
|
console.warn('Invalid date provided to safeToISOString, using current time as fallback');
|
|
return new Date().toISOString();
|
|
}
|
|
return date.toISOString();
|
|
} catch (error) {
|
|
console.warn('Error converting date to ISO string:', error instanceof Error ? error.message : String(error));
|
|
return new Date().toISOString();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Load auto-sync settings from .env file
|
|
*/
|
|
loadSettings() {
|
|
try {
|
|
const envPath = join(process.cwd(), '.env');
|
|
const envContent = readFileSync(envPath, 'utf8');
|
|
|
|
const settings = {
|
|
autoSyncEnabled: false,
|
|
syncIntervalType: 'predefined',
|
|
syncIntervalPredefined: '1hour',
|
|
syncIntervalCron: '',
|
|
autoDownloadNew: false,
|
|
autoUpdateExisting: false,
|
|
notificationEnabled: false,
|
|
appriseUrls: [],
|
|
lastAutoSync: ''
|
|
};
|
|
const lines = envContent.split('\n');
|
|
|
|
for (const line of lines) {
|
|
const [key, ...valueParts] = line.split('=');
|
|
if (key && valueParts.length > 0) {
|
|
let value = valueParts.join('=').trim();
|
|
// Remove surrounding quotes if present
|
|
if (value.startsWith('"') && value.endsWith('"')) {
|
|
value = value.slice(1, -1);
|
|
}
|
|
|
|
switch (key.trim()) {
|
|
case 'AUTO_SYNC_ENABLED':
|
|
settings.autoSyncEnabled = value === 'true';
|
|
break;
|
|
case 'SYNC_INTERVAL_TYPE':
|
|
settings.syncIntervalType = value;
|
|
break;
|
|
case 'SYNC_INTERVAL_PREDEFINED':
|
|
settings.syncIntervalPredefined = value;
|
|
break;
|
|
case 'SYNC_INTERVAL_CRON':
|
|
settings.syncIntervalCron = value;
|
|
break;
|
|
case 'AUTO_DOWNLOAD_NEW':
|
|
settings.autoDownloadNew = value === 'true';
|
|
break;
|
|
case 'AUTO_UPDATE_EXISTING':
|
|
settings.autoUpdateExisting = value === 'true';
|
|
break;
|
|
case 'NOTIFICATION_ENABLED':
|
|
settings.notificationEnabled = value === 'true';
|
|
break;
|
|
case 'APPRISE_URLS':
|
|
try {
|
|
settings.appriseUrls = JSON.parse(value || '[]');
|
|
} catch {
|
|
settings.appriseUrls = [];
|
|
}
|
|
break;
|
|
case 'LAST_AUTO_SYNC':
|
|
settings.lastAutoSync = value;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
return settings;
|
|
} catch (error) {
|
|
console.error('Error loading auto-sync settings:', error);
|
|
return {
|
|
autoSyncEnabled: false,
|
|
syncIntervalType: 'predefined',
|
|
syncIntervalPredefined: '1hour',
|
|
syncIntervalCron: '',
|
|
autoDownloadNew: false,
|
|
autoUpdateExisting: false,
|
|
notificationEnabled: false,
|
|
appriseUrls: [],
|
|
lastAutoSync: ''
|
|
};
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Save auto-sync settings to .env file
|
|
* @param {Object} settings - Settings object
|
|
* @param {boolean} settings.autoSyncEnabled
|
|
* @param {string} settings.syncIntervalType
|
|
* @param {string} [settings.syncIntervalPredefined]
|
|
* @param {string} [settings.syncIntervalCron]
|
|
* @param {boolean} settings.autoDownloadNew
|
|
* @param {boolean} settings.autoUpdateExisting
|
|
* @param {boolean} settings.notificationEnabled
|
|
* @param {Array<string>} [settings.appriseUrls]
|
|
* @param {string} [settings.lastAutoSync]
|
|
*/
|
|
saveSettings(settings) {
|
|
try {
|
|
const envPath = join(process.cwd(), '.env');
|
|
let envContent = '';
|
|
|
|
try {
|
|
envContent = readFileSync(envPath, 'utf8');
|
|
} catch {
|
|
// .env file doesn't exist, create it
|
|
}
|
|
|
|
const lines = envContent.split('\n');
|
|
const newLines = [];
|
|
const settingsMap = {
|
|
'AUTO_SYNC_ENABLED': settings.autoSyncEnabled.toString(),
|
|
'SYNC_INTERVAL_TYPE': settings.syncIntervalType,
|
|
'SYNC_INTERVAL_PREDEFINED': settings.syncIntervalPredefined || '',
|
|
'SYNC_INTERVAL_CRON': settings.syncIntervalCron || '',
|
|
'AUTO_DOWNLOAD_NEW': settings.autoDownloadNew.toString(),
|
|
'AUTO_UPDATE_EXISTING': settings.autoUpdateExisting.toString(),
|
|
'NOTIFICATION_ENABLED': settings.notificationEnabled.toString(),
|
|
'APPRISE_URLS': JSON.stringify(settings.appriseUrls || []),
|
|
'LAST_AUTO_SYNC': settings.lastAutoSync || ''
|
|
};
|
|
|
|
const existingKeys = new Set();
|
|
|
|
for (const line of lines) {
|
|
const [key] = line.split('=');
|
|
const trimmedKey = key?.trim();
|
|
if (trimmedKey && trimmedKey in settingsMap) {
|
|
// @ts-ignore - Dynamic key access is safe here
|
|
newLines.push(`${trimmedKey}=${settingsMap[trimmedKey]}`);
|
|
existingKeys.add(trimmedKey);
|
|
} else if (trimmedKey && !(trimmedKey in settingsMap)) {
|
|
newLines.push(line);
|
|
}
|
|
}
|
|
|
|
// Add any missing settings
|
|
for (const [key, value] of Object.entries(settingsMap)) {
|
|
if (!existingKeys.has(key)) {
|
|
newLines.push(`${key}=${value}`);
|
|
}
|
|
}
|
|
|
|
writeFileSync(envPath, newLines.join('\n'));
|
|
console.log('Auto-sync settings saved successfully');
|
|
} catch (error) {
|
|
console.error('Error saving auto-sync settings:', error);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Schedule auto-sync cron job
|
|
*/
|
|
scheduleAutoSync() {
|
|
this.stopAutoSync(); // Stop any existing job
|
|
|
|
const settings = this.loadSettings();
|
|
if (!settings.autoSyncEnabled) {
|
|
return;
|
|
}
|
|
|
|
let cronExpression;
|
|
|
|
if (settings.syncIntervalType === 'custom') {
|
|
cronExpression = settings.syncIntervalCron;
|
|
} else {
|
|
// Convert predefined intervals to cron expressions
|
|
const intervalMap = {
|
|
'15min': '*/15 * * * *',
|
|
'30min': '*/30 * * * *',
|
|
'1hour': '0 * * * *',
|
|
'6hours': '0 */6 * * *',
|
|
'12hours': '0 */12 * * *',
|
|
'24hours': '0 0 * * *'
|
|
};
|
|
// @ts-ignore - Dynamic key access is safe here
|
|
cronExpression = intervalMap[settings.syncIntervalPredefined] || '0 * * * *';
|
|
}
|
|
|
|
// Validate cron expression (5-field format for node-cron)
|
|
if (!cronValidator.isValidCron(cronExpression, { seconds: false })) {
|
|
console.error('Invalid cron expression:', cronExpression);
|
|
return;
|
|
}
|
|
|
|
console.log(`Scheduling auto-sync with cron expression: ${cronExpression}`);
|
|
|
|
this.cronJob = cron.schedule(cronExpression, async () => {
|
|
if (this.isRunning) {
|
|
console.log('Auto-sync already running, skipping...');
|
|
return;
|
|
}
|
|
|
|
console.log('Starting scheduled auto-sync...');
|
|
await this.executeAutoSync();
|
|
}, {
|
|
scheduled: true,
|
|
timezone: 'UTC'
|
|
});
|
|
|
|
console.log('Auto-sync cron job scheduled successfully');
|
|
}
|
|
|
|
/**
|
|
* Stop auto-sync cron job
|
|
*/
|
|
stopAutoSync() {
|
|
if (this.cronJob) {
|
|
this.cronJob.stop();
|
|
this.cronJob = null;
|
|
console.log('Auto-sync cron job stopped');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Execute auto-sync process
|
|
*/
|
|
async executeAutoSync() {
|
|
if (this.isRunning) {
|
|
console.log('Auto-sync already running, skipping...');
|
|
return { success: false, message: 'Auto-sync already running' };
|
|
}
|
|
|
|
this.isRunning = true;
|
|
const startTime = new Date();
|
|
|
|
try {
|
|
console.log('Starting auto-sync execution...');
|
|
|
|
// Step 1: Sync JSON files
|
|
console.log('Syncing JSON files...');
|
|
const syncResult = await githubJsonService.syncJsonFiles();
|
|
|
|
if (!syncResult.success) {
|
|
throw new Error(`JSON sync failed: ${syncResult.message}`);
|
|
}
|
|
|
|
const results = {
|
|
jsonSync: syncResult,
|
|
newScripts: /** @type {string[]} */ ([]),
|
|
updatedScripts: /** @type {string[]} */ ([]),
|
|
errors: /** @type {string[]} */ ([])
|
|
};
|
|
|
|
// Step 2: Auto-download/update scripts if enabled
|
|
const settings = this.loadSettings();
|
|
|
|
if (settings.autoDownloadNew || settings.autoUpdateExisting) {
|
|
console.log('Processing synced JSON files for script downloads...');
|
|
|
|
// Only process scripts for files that were actually synced
|
|
if (syncResult.syncedFiles && syncResult.syncedFiles.length > 0) {
|
|
console.log(`Processing ${syncResult.syncedFiles.length} synced JSON files for script downloads...`);
|
|
|
|
// Get scripts only for the synced files
|
|
const localScriptsService = await import('./localScripts.js');
|
|
const syncedScripts = [];
|
|
|
|
for (const filename of syncResult.syncedFiles) {
|
|
try {
|
|
// Extract slug from filename (remove .json extension)
|
|
const slug = filename.replace('.json', '');
|
|
const script = await localScriptsService.localScriptsService.getScriptBySlug(slug);
|
|
if (script) {
|
|
syncedScripts.push(script);
|
|
}
|
|
} catch (error) {
|
|
console.warn(`Error loading script from ${filename}:`, error);
|
|
}
|
|
}
|
|
|
|
console.log(`Found ${syncedScripts.length} scripts from synced JSON files`);
|
|
|
|
// Filter to only truly NEW scripts (not previously downloaded)
|
|
const newScripts = [];
|
|
const existingScripts = [];
|
|
|
|
for (const script of syncedScripts) {
|
|
try {
|
|
// Validate script object
|
|
if (!script || !script.slug) {
|
|
console.warn('Invalid script object found, skipping:', script);
|
|
continue;
|
|
}
|
|
|
|
const isDownloaded = await scriptDownloaderService.isScriptDownloaded(script);
|
|
if (!isDownloaded) {
|
|
newScripts.push(script);
|
|
} else {
|
|
existingScripts.push(script);
|
|
}
|
|
} catch (error) {
|
|
console.warn(`Error checking script ${script?.slug || 'unknown'}:`, error);
|
|
// Treat as new script if we can't check
|
|
if (script && script.slug) {
|
|
newScripts.push(script);
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log(`Found ${newScripts.length} new scripts and ${existingScripts.length} existing scripts from synced files`);
|
|
|
|
// Download new scripts
|
|
if (settings.autoDownloadNew && newScripts.length > 0) {
|
|
console.log(`Auto-downloading ${newScripts.length} new scripts...`);
|
|
const downloaded = [];
|
|
const errors = [];
|
|
|
|
for (const script of newScripts) {
|
|
try {
|
|
const result = await scriptDownloaderService.loadScript(script);
|
|
if (result.success) {
|
|
downloaded.push(script.name || script.slug);
|
|
console.log(`Downloaded script: ${script.name || script.slug}`);
|
|
} else {
|
|
errors.push(`${script.name || script.slug}: ${result.message}`);
|
|
}
|
|
} catch (error) {
|
|
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
errors.push(`${script.name || script.slug}: ${errorMsg}`);
|
|
console.error(`Failed to download script ${script.slug}:`, error);
|
|
}
|
|
}
|
|
|
|
results.newScripts = downloaded;
|
|
results.errors.push(...errors);
|
|
}
|
|
|
|
// Update existing scripts
|
|
if (settings.autoUpdateExisting && existingScripts.length > 0) {
|
|
console.log(`Auto-updating ${existingScripts.length} existing scripts...`);
|
|
const updated = [];
|
|
const errors = [];
|
|
|
|
for (const script of existingScripts) {
|
|
try {
|
|
// Always update existing scripts when auto-update is enabled
|
|
const result = await scriptDownloaderService.loadScript(script);
|
|
if (result.success) {
|
|
updated.push(script.name || script.slug);
|
|
console.log(`Updated script: ${script.name || script.slug}`);
|
|
} else {
|
|
errors.push(`${script.name || script.slug}: ${result.message}`);
|
|
}
|
|
} catch (error) {
|
|
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
errors.push(`${script.name || script.slug}: ${errorMsg}`);
|
|
console.error(`Failed to update script ${script.slug}:`, error);
|
|
}
|
|
}
|
|
|
|
results.updatedScripts = updated;
|
|
results.errors.push(...errors);
|
|
}
|
|
} else {
|
|
console.log('No JSON files were synced, skipping script download/update');
|
|
}
|
|
} else {
|
|
console.log('Auto-download/update disabled, skipping script processing');
|
|
}
|
|
|
|
// Step 3: Send notifications if enabled
|
|
if (settings.notificationEnabled && settings.appriseUrls?.length > 0) {
|
|
console.log('Sending notifications...');
|
|
await this.sendSyncNotification(results);
|
|
}
|
|
|
|
// Step 4: Update last sync time
|
|
const lastSyncTime = this.safeToISOString(new Date());
|
|
const updatedSettings = { ...settings, lastAutoSync: lastSyncTime };
|
|
this.saveSettings(updatedSettings);
|
|
|
|
const duration = new Date().getTime() - startTime.getTime();
|
|
console.log(`Auto-sync completed successfully in ${duration}ms`);
|
|
|
|
return {
|
|
success: true,
|
|
message: 'Auto-sync completed successfully',
|
|
results,
|
|
duration
|
|
};
|
|
|
|
} catch (error) {
|
|
console.error('Auto-sync execution failed:', error);
|
|
|
|
// Send error notification if enabled
|
|
const settings = this.loadSettings();
|
|
if (settings.notificationEnabled && settings.appriseUrls?.length > 0) {
|
|
try {
|
|
await appriseService.sendNotification(
|
|
'Auto-Sync Failed',
|
|
`Auto-sync failed with error: ${error instanceof Error ? error.message : String(error)}`,
|
|
settings.appriseUrls
|
|
);
|
|
} catch (notifError) {
|
|
console.error('Failed to send error notification:', notifError);
|
|
}
|
|
}
|
|
|
|
return {
|
|
success: false,
|
|
message: error instanceof Error ? error.message : String(error),
|
|
error: error instanceof Error ? error.message : String(error)
|
|
};
|
|
} finally {
|
|
this.isRunning = false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Load categories from metadata.json
|
|
*/
|
|
loadCategories() {
|
|
try {
|
|
const metadataPath = join(process.cwd(), 'scripts', 'json', 'metadata.json');
|
|
const metadataContent = readFileSync(metadataPath, 'utf8');
|
|
const metadata = JSON.parse(metadataContent);
|
|
return metadata.categories || [];
|
|
} catch (error) {
|
|
console.error('Error loading categories:', error);
|
|
return [];
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Group scripts by category
|
|
* @param {Array<any>} scripts - Array of script objects
|
|
* @param {Array<any>} categories - Array of category objects
|
|
*/
|
|
groupScriptsByCategory(scripts, categories) {
|
|
const categoryMap = new Map();
|
|
categories.forEach(cat => categoryMap.set(cat.id, cat.name));
|
|
|
|
const grouped = new Map();
|
|
|
|
scripts.forEach(script => {
|
|
// Validate script object
|
|
if (!script || !script.name) {
|
|
console.warn('Invalid script object in groupScriptsByCategory, skipping:', script);
|
|
return;
|
|
}
|
|
|
|
const scriptCategories = script.categories || [0]; // Default to Miscellaneous (id: 0)
|
|
scriptCategories.forEach((/** @type {number} */ catId) => {
|
|
const categoryName = categoryMap.get(catId) || 'Miscellaneous';
|
|
if (!grouped.has(categoryName)) {
|
|
grouped.set(categoryName, []);
|
|
}
|
|
grouped.get(categoryName).push(script.name);
|
|
});
|
|
});
|
|
|
|
return grouped;
|
|
}
|
|
|
|
/**
|
|
* Send notification about sync results
|
|
* @param {Object} results - Sync results object
|
|
*/
|
|
async sendSyncNotification(results) {
|
|
const settings = this.loadSettings();
|
|
|
|
if (!settings.notificationEnabled || !settings.appriseUrls?.length) {
|
|
return;
|
|
}
|
|
|
|
const title = 'ProxmoxVE-Local - Auto-Sync Completed';
|
|
let body = `Auto-sync completed successfully.\n\n`;
|
|
|
|
// Add JSON sync info
|
|
// @ts-ignore - Dynamic property access
|
|
if (results.jsonSync) {
|
|
// @ts-ignore - Dynamic property access
|
|
body += `JSON Files: ${results.jsonSync.syncedCount} synced, ${results.jsonSync.skippedCount} up-to-date\n`;
|
|
// @ts-ignore - Dynamic property access
|
|
if (results.jsonSync.errors?.length > 0) {
|
|
// @ts-ignore - Dynamic property access
|
|
body += `JSON Errors: ${results.jsonSync.errors.length}\n`;
|
|
}
|
|
body += '\n';
|
|
}
|
|
|
|
// Load categories for grouping
|
|
const categories = this.loadCategories();
|
|
|
|
// @ts-ignore - Dynamic property access
|
|
if (results.newScripts?.length > 0) {
|
|
// @ts-ignore - Dynamic property access
|
|
body += `New scripts downloaded: ${results.newScripts.length}\n`;
|
|
|
|
// Group new scripts by category
|
|
// @ts-ignore - Dynamic property access
|
|
const newScriptsGrouped = this.groupScriptsByCategory(results.newScripts, categories);
|
|
|
|
// Sort categories by name for consistent ordering
|
|
const sortedCategories = Array.from(newScriptsGrouped.keys()).sort();
|
|
|
|
sortedCategories.forEach(categoryName => {
|
|
const scripts = newScriptsGrouped.get(categoryName);
|
|
body += `\n**${categoryName}:**\n`;
|
|
scripts.forEach((/** @type {string} */ scriptName) => {
|
|
body += `• ${scriptName}\n`;
|
|
});
|
|
});
|
|
body += '\n';
|
|
}
|
|
|
|
// @ts-ignore - Dynamic property access
|
|
if (results.updatedScripts?.length > 0) {
|
|
// @ts-ignore - Dynamic property access
|
|
body += `Scripts updated: ${results.updatedScripts.length}\n`;
|
|
|
|
// Group updated scripts by category
|
|
// @ts-ignore - Dynamic property access
|
|
const updatedScriptsGrouped = this.groupScriptsByCategory(results.updatedScripts, categories);
|
|
|
|
// Sort categories by name for consistent ordering
|
|
const sortedCategories = Array.from(updatedScriptsGrouped.keys()).sort();
|
|
|
|
sortedCategories.forEach(categoryName => {
|
|
const scripts = updatedScriptsGrouped.get(categoryName);
|
|
body += `\n**${categoryName}:**\n`;
|
|
scripts.forEach((/** @type {string} */ scriptName) => {
|
|
body += `• ${scriptName}\n`;
|
|
});
|
|
});
|
|
body += '\n';
|
|
}
|
|
|
|
// @ts-ignore - Dynamic property access
|
|
if (results.errors?.length > 0) {
|
|
// @ts-ignore - Dynamic property access
|
|
body += `Script errors encountered: ${results.errors.length}\n`;
|
|
// @ts-ignore - Dynamic property access
|
|
body += `• ${results.errors.slice(0, 5).join('\n• ')}\n`;
|
|
// @ts-ignore - Dynamic property access
|
|
if (results.errors.length > 5) {
|
|
// @ts-ignore - Dynamic property access
|
|
body += `• ... and ${results.errors.length - 5} more errors\n`;
|
|
}
|
|
}
|
|
|
|
// @ts-ignore - Dynamic property access
|
|
if (results.newScripts?.length === 0 && results.updatedScripts?.length === 0 && results.errors?.length === 0) {
|
|
body += 'No script changes detected.';
|
|
}
|
|
|
|
try {
|
|
await appriseService.sendNotification(title, body, settings.appriseUrls);
|
|
console.log('Sync notification sent successfully');
|
|
} catch (error) {
|
|
console.error('Failed to send sync notification:', error);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Test notification
|
|
*/
|
|
async testNotification() {
|
|
const settings = this.loadSettings();
|
|
|
|
if (!settings.notificationEnabled || !settings.appriseUrls?.length) {
|
|
return {
|
|
success: false,
|
|
message: 'Notifications not enabled or no Apprise URLs configured'
|
|
};
|
|
}
|
|
|
|
try {
|
|
await appriseService.sendNotification(
|
|
'ProxmoxVE-Local - Test Notification',
|
|
'This is a test notification from PVE Scripts Local auto-sync feature.',
|
|
settings.appriseUrls
|
|
);
|
|
|
|
return {
|
|
success: true,
|
|
message: 'Test notification sent successfully'
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
success: false,
|
|
message: `Failed to send test notification: ${error instanceof Error ? error.message : String(error)}`
|
|
};
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get auto-sync status
|
|
*/
|
|
getStatus() {
|
|
return {
|
|
isRunning: this.isRunning,
|
|
hasCronJob: !!this.cronJob,
|
|
lastSync: this.loadSettings().lastAutoSync
|
|
};
|
|
}
|
|
}
|