feat: Add comprehensive auto-sync functionality
✨ New Features: - Auto-sync service with configurable intervals (15min, 30min, 1hour, 6hours, 12hours, 24hours, custom cron) - Automatic JSON file synchronization from GitHub repositories - Auto-download new scripts when JSON files are updated - Auto-update existing scripts when newer versions are available - Apprise notification service integration for sync status updates - Comprehensive error handling and logging 🔧 Technical Implementation: - AutoSyncService: Core scheduling and execution logic - GitHubJsonService: Handles JSON file synchronization from GitHub - AppriseService: Sends notifications via multiple channels (Discord, Telegram, Email, Slack, etc.) - ScriptDownloaderService: Manages automatic script downloads and updates - Settings API: RESTful endpoints for auto-sync configuration - UI Integration: Settings modal with auto-sync configuration options 📋 Configuration Options: - Enable/disable auto-sync functionality - Flexible scheduling (predefined intervals or custom cron expressions) - Selective script processing (new downloads, updates, or both) - Notification settings with multiple Apprise URL support - Environment-based configuration with .env file persistence 🎯 Benefits: - Keeps script repository automatically synchronized - Reduces manual maintenance overhead - Provides real-time notifications of sync status - Supports multiple notification channels - Configurable to match different deployment needs This feature significantly enhances the automation capabilities of PVE Scripts Local, making it a truly hands-off solution for script management.
This commit is contained in:
Michel Roegl-Brunner
271
src/server/services/githubJsonService.js
Normal file
271
src/server/services/githubJsonService.js
Normal file
@@ -0,0 +1,271 @@
|
||||
import { writeFile, mkdir } from 'fs/promises';
|
||||
import { readFileSync, readdirSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
export class GitHubJsonService {
|
||||
constructor() {
|
||||
this.baseUrl = null;
|
||||
this.repoUrl = null;
|
||||
this.branch = null;
|
||||
this.jsonFolder = null;
|
||||
this.localJsonDirectory = null;
|
||||
this.scriptCache = new Map();
|
||||
}
|
||||
|
||||
initializeConfig() {
|
||||
if (this.repoUrl === null) {
|
||||
// Get environment variables
|
||||
this.repoUrl = process.env.REPO_URL || "";
|
||||
this.branch = process.env.REPO_BRANCH || "main";
|
||||
this.jsonFolder = process.env.JSON_FOLDER || "scripts";
|
||||
this.localJsonDirectory = join(process.cwd(), 'scripts', 'json');
|
||||
|
||||
// Only validate GitHub URL if it's provided
|
||||
if (this.repoUrl) {
|
||||
// Extract owner and repo from the URL
|
||||
const urlMatch = /github\.com\/([^\/]+)\/([^\/]+)/.exec(this.repoUrl);
|
||||
if (!urlMatch) {
|
||||
throw new Error(`Invalid GitHub repository URL: ${this.repoUrl}`);
|
||||
}
|
||||
|
||||
const [, owner, repo] = urlMatch;
|
||||
this.baseUrl = `https://api.github.com/repos/${owner}/${repo}`;
|
||||
} else {
|
||||
// Set a dummy base URL if no REPO_URL is provided
|
||||
this.baseUrl = "";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fetchFromGitHub(endpoint) {
|
||||
this.initializeConfig();
|
||||
const response = await fetch(`${this.baseUrl}${endpoint}`, {
|
||||
headers: {
|
||||
'Accept': 'application/vnd.github.v3+json',
|
||||
'User-Agent': 'PVEScripts-Local/1.0',
|
||||
...(process.env.GITHUB_TOKEN && { 'Authorization': `token ${process.env.GITHUB_TOKEN}` })
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`GitHub API error: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async syncJsonFiles() {
|
||||
try {
|
||||
this.initializeConfig();
|
||||
|
||||
if (!this.baseUrl) {
|
||||
return {
|
||||
success: false,
|
||||
message: 'No GitHub repository configured'
|
||||
};
|
||||
}
|
||||
|
||||
console.log('Starting fast incremental JSON sync...');
|
||||
|
||||
// Ensure local directory exists
|
||||
await mkdir(this.localJsonDirectory, { recursive: true });
|
||||
|
||||
// Step 1: Get file list from GitHub (single API call)
|
||||
console.log('Fetching file list from GitHub...');
|
||||
const files = await this.fetchFromGitHub(`/contents/${this.jsonFolder}?ref=${this.branch}`);
|
||||
|
||||
if (!Array.isArray(files)) {
|
||||
throw new Error('Invalid response from GitHub API');
|
||||
}
|
||||
|
||||
const jsonFiles = files.filter(file => file.name.endsWith('.json'));
|
||||
console.log(`Found ${jsonFiles.length} JSON files in repository`);
|
||||
|
||||
// Step 2: Get local file list (fast local operation)
|
||||
const localFiles = new Map();
|
||||
try {
|
||||
const localFileList = readdirSync(this.localJsonDirectory);
|
||||
for (const fileName of localFileList) {
|
||||
if (fileName.endsWith('.json')) {
|
||||
const filePath = join(this.localJsonDirectory, fileName);
|
||||
const stats = require('fs').statSync(filePath);
|
||||
localFiles.set(fileName, {
|
||||
mtime: stats.mtime,
|
||||
size: stats.size
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('No local files found, will download all');
|
||||
}
|
||||
|
||||
console.log(`Found ${localFiles.size} local JSON files`);
|
||||
|
||||
// Step 3: Compare and identify files that need syncing
|
||||
const filesToSync = [];
|
||||
let skippedCount = 0;
|
||||
|
||||
for (const file of jsonFiles) {
|
||||
const localFile = localFiles.get(file.name);
|
||||
|
||||
if (!localFile) {
|
||||
// File doesn't exist locally
|
||||
filesToSync.push(file);
|
||||
console.log(`Missing: ${file.name}`);
|
||||
} else {
|
||||
// Compare modification times and sizes
|
||||
const localMtime = new Date(localFile.mtime);
|
||||
const remoteMtime = new Date(file.updated_at);
|
||||
const localSize = localFile.size;
|
||||
const remoteSize = file.size;
|
||||
|
||||
// Sync if remote is newer OR sizes are different (content changed)
|
||||
if (localMtime < remoteMtime || localSize !== remoteSize) {
|
||||
filesToSync.push(file);
|
||||
console.log(`Changed: ${file.name} (${localMtime.toISOString()} -> ${remoteMtime.toISOString()})`);
|
||||
} else {
|
||||
skippedCount++;
|
||||
console.log(`Up-to-date: ${file.name}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Files to sync: ${filesToSync.length}, Up-to-date: ${skippedCount}`);
|
||||
|
||||
// Step 4: Download only the files that need syncing
|
||||
let syncedCount = 0;
|
||||
const errors = [];
|
||||
const syncedFiles = [];
|
||||
|
||||
// Process files in batches to avoid overwhelming the API
|
||||
const batchSize = 10;
|
||||
for (let i = 0; i < filesToSync.length; i += batchSize) {
|
||||
const batch = filesToSync.slice(i, i + batchSize);
|
||||
|
||||
// Process batch in parallel
|
||||
const promises = batch.map(async (file) => {
|
||||
try {
|
||||
const content = await this.fetchFromGitHub(`/contents/${file.path}?ref=${this.branch}`);
|
||||
|
||||
if (content.content) {
|
||||
// Decode base64 content
|
||||
const fileContent = Buffer.from(content.content, 'base64').toString('utf-8');
|
||||
|
||||
// Write to local file
|
||||
const localPath = join(this.localJsonDirectory, file.name);
|
||||
await writeFile(localPath, fileContent, 'utf-8');
|
||||
|
||||
// Update file modification time to match remote
|
||||
const remoteMtime = new Date(file.updated_at);
|
||||
require('fs').utimesSync(localPath, remoteMtime, remoteMtime);
|
||||
|
||||
syncedCount++;
|
||||
syncedFiles.push(file.name);
|
||||
console.log(`Synced: ${file.name}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to sync ${file.name}:`, error.message);
|
||||
errors.push(`${file.name}: ${error.message}`);
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(promises);
|
||||
|
||||
// Small delay between batches to be nice to the API
|
||||
if (i + batchSize < filesToSync.length) {
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`JSON sync completed. Synced ${syncedCount} files, skipped ${skippedCount} files.`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
message: `Successfully synced ${syncedCount} JSON files (${skippedCount} up-to-date)`,
|
||||
syncedCount,
|
||||
skippedCount,
|
||||
syncedFiles,
|
||||
errors
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
console.error('JSON sync failed:', error);
|
||||
return {
|
||||
success: false,
|
||||
message: error.message,
|
||||
error: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async getAllScripts() {
|
||||
try {
|
||||
this.initializeConfig();
|
||||
|
||||
if (!this.localJsonDirectory) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const scripts = [];
|
||||
|
||||
// Read all JSON files from local directory
|
||||
const files = readdirSync(this.localJsonDirectory);
|
||||
const jsonFiles = files.filter(file => file.endsWith('.json'));
|
||||
|
||||
for (const file of jsonFiles) {
|
||||
try {
|
||||
const filePath = join(this.localJsonDirectory, file);
|
||||
const content = readFileSync(filePath, 'utf-8');
|
||||
const script = JSON.parse(content);
|
||||
|
||||
if (script && typeof script === 'object') {
|
||||
scripts.push(script);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to parse ${file}:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
return scripts;
|
||||
} catch (error) {
|
||||
console.error('Failed to get all scripts:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get scripts only for specific JSON files that were synced
|
||||
*/
|
||||
async getScriptsForFiles(syncedFiles) {
|
||||
try {
|
||||
this.initializeConfig();
|
||||
|
||||
if (!this.localJsonDirectory || !syncedFiles || syncedFiles.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const scripts = [];
|
||||
|
||||
for (const fileName of syncedFiles) {
|
||||
try {
|
||||
const filePath = join(this.localJsonDirectory, fileName);
|
||||
const content = readFileSync(filePath, 'utf-8');
|
||||
const script = JSON.parse(content);
|
||||
|
||||
if (script && typeof script === 'object') {
|
||||
scripts.push(script);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to parse ${fileName}:`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
return scripts;
|
||||
} catch (error) {
|
||||
console.error('Failed to get scripts for synced files:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const githubJsonService = new GitHubJsonService();
|
||||
Reference in New Issue
Block a user