diff --git a/scripts/.env.test b/scripts/.env.test new file mode 100644 index 0000000..eeeec36 --- /dev/null +++ b/scripts/.env.test @@ -0,0 +1,5 @@ +DB_HOST=db.psg.net.au +DB_PORT=3307 +DB_USER=svc_sysinfo +DB_PASSWORD=2pT08pEuxqFiN6eD348vBlgoMfyfOjGB +DB_NAME=db_ld-spring-backend diff --git a/scripts/.last_synced_date b/scripts/.last_synced_date deleted file mode 100644 index aa022a2..0000000 --- a/scripts/.last_synced_date +++ /dev/null @@ -1 +0,0 @@ -2001-08-20T00:00:00.000Z \ No newline at end of file diff --git a/scripts/cve-sync.log b/scripts/cve-sync.log index e78b6e9..b9ab968 100644 --- a/scripts/cve-sync.log +++ b/scripts/cve-sync.log @@ -98,3 +98,31 @@ [08 Oct 2025, 09:22:22 am] โœ… CVE import complete! [08 Oct 2025, 09:22:22 am] โœ… CVE import complete! [08 Oct 2025, 01:22:22 am] โœ… fetchCVE.js finished with exit code: 0 +[08 Oct 2025, 02:21:34 am] ๐Ÿš€ ๐Ÿ“ก CVE backfill launched - will sync back to 2002. +[08 Oct 2025, 10:21:35 am] ๐Ÿงช Getting CVEs from the last 30 of days +[08 Oct 2025, 10:21:35 am] ๐Ÿงช Getting CVEs from the last 30 of days +[08 Oct 2025, 10:21:35 am] ๐Ÿ” Resuming CVE backfill from 20 Aug 2001 +[08 Oct 2025, 10:21:35 am] ๐Ÿ“ก Fetching published CVEs from 23 Apr 2001 to 20 Aug 2001... +[08 Oct 2025, 10:21:35 am] ๐Ÿ” Resuming CVE backfill from 20 Aug 2001 +[08 Oct 2025, 10:21:35 am] ๐Ÿ“ก Fetching published CVEs from 23 Apr 2001 to 20 Aug 2001... +[08 Oct 2025, 10:21:36 am] ๐Ÿ“„ Page 1 โ€” 594 CVEs from index 0 +[08 Oct 2025, 10:21:36 am] ๐Ÿ“„ Page 1 โ€” 594 CVEs from index 0 +[08 Oct 2025, 10:21:49 am] ๐Ÿ›‘ Reached earliest supported CVE publication date โ€” halting backfill. +[08 Oct 2025, 10:21:49 am] โœ… CVE backfill complete! +[08 Oct 2025, 10:21:49 am] ๐Ÿ›‘ Reached earliest supported CVE publication date โ€” halting backfill. +[08 Oct 2025, 10:21:49 am] โœ… CVE backfill complete! +[08 Oct 2025, 02:21:49 am] โœ… fetchCVE_withMORE.js finished with exit code: 0 +[08 Oct 2025, 02:22:45 am] ๐Ÿš€ ๐Ÿ“ก CVE backfill launched - will sync back to 2002. +[08 Oct 2025, 10:22:45 am] ๐Ÿงช Getting CVEs from the last 30 of days +[08 Oct 2025, 10:22:45 am] ๐Ÿงช Getting CVEs from the last 30 of days +[08 Oct 2025, 10:22:45 am] ๐Ÿ” Resuming CVE backfill from 23 Apr 2001 +[08 Oct 2025, 10:22:45 am] ๐Ÿ“ก Fetching published CVEs from 25 Dec 2000 to 23 Apr 2001... +[08 Oct 2025, 10:22:45 am] ๐Ÿ” Resuming CVE backfill from 23 Apr 2001 +[08 Oct 2025, 10:22:45 am] ๐Ÿ“ก Fetching published CVEs from 25 Dec 2000 to 23 Apr 2001... +[08 Oct 2025, 10:22:47 am] ๐Ÿ“„ Page 1 โ€” 389 CVEs from index 0 +[08 Oct 2025, 10:22:47 am] ๐Ÿ“„ Page 1 โ€” 389 CVEs from index 0 +[08 Oct 2025, 10:22:57 am] ๐Ÿ›‘ Reached earliest supported CVE publication date โ€” halting backfill. +[08 Oct 2025, 10:22:57 am] โœ… CVE backfill complete! +[08 Oct 2025, 10:22:57 am] ๐Ÿ›‘ Reached earliest supported CVE publication date โ€” halting backfill. +[08 Oct 2025, 10:22:57 am] โœ… CVE backfill complete! +[08 Oct 2025, 02:22:57 am] โœ… fetchCVE_withMORE.js finished with exit code: 0 diff --git a/scripts/cvelistV5 b/scripts/cvelistV5 new file mode 160000 index 0000000..d18b6e1 --- /dev/null +++ b/scripts/cvelistV5 @@ -0,0 +1 @@ +Subproject commit d18b6e1ab00a0f5c5dbe40521a105dfe3424543f diff --git a/scripts/fetchCVE_withMORE.js b/scripts/fetchCVE_withMORE.js index 78e5778..eeeed67 100644 --- a/scripts/fetchCVE_withMORE.js +++ b/scripts/fetchCVE_withMORE.js @@ -223,21 +223,28 @@ async function importCVEFeed() { } async function importCVEFeedBackfill() { - const now = new Date(); - const resumeFrom = loadLastSyncedDate(); - let startFrom = resumeFrom ? new Date(resumeFrom) : now; - + const EARLIEST_CVE_DATE = new Date('2002-01-01T00:00:00.000Z'); const MAX_RANGE_DAYS = 120; + const resumeFrom = loadLastSyncedDate(); + let currentStart = resumeFrom ? new Date(resumeFrom) : EARLIEST_CVE_DATE; + log(resumeFrom - ? `๐Ÿ” Resuming CVE backfill from ${formatShortDate(startFrom.toISOString())}` - : `โฎ๏ธ Starting CVE backfill from today (${formatShortDate(startFrom.toISOString())})` + ? `๐Ÿ” Resuming CVE backfill from ${formatShortDate(currentStart.toISOString())}` + : `โฎ๏ธ Starting CVE backfill from ${formatShortDate(EARLIEST_CVE_DATE.toISOString())}` ); - while (true) { - const end = new Date(startFrom); - const start = new Date(startFrom); - start.setDate(start.getDate() - MAX_RANGE_DAYS + 1); // 120-day window + const now = new Date(); + + while (currentStart < now) { + const start = new Date(currentStart); + const end = new Date(currentStart); + end.setDate(end.getDate() + MAX_RANGE_DAYS - 1); // 120-day window + + // Don't go past today + if (end > now) { + end.setTime(now.getTime()); + } const startISO = start.toISOString(); const endISO = end.toISOString(); @@ -260,7 +267,7 @@ async function importCVEFeedBackfill() { break; } - log(`๐Ÿ“„ Page ${++pageCount} โ€” ${vulnerabilities.length} CVEs from index ${startIndex}`); + log(`๐Ÿ“„ Page ${++pageCount} โ€” ${vulnerabilities.length} CVEs from index ${startIndex} of ~${totalResults}`); for (const vuln of vulnerabilities) { await processCVE(vuln); @@ -270,16 +277,20 @@ async function importCVEFeedBackfill() { await new Promise((r) => setTimeout(r, 6000)); } while (startIndex < totalResults); - // Move the window backward - saveLastSyncedDate(start.toISOString()); - startFrom = start; + // Move the window forward + currentStart.setDate(currentStart.getDate() + MAX_RANGE_DAYS); + saveLastSyncedDate(currentStart.toISOString()); + + log(`โœ… Completed ${humanRange}. Next start: ${formatShortDate(currentStart.toISOString())}`); } catch (err) { log(`โŒ Error during ${humanRange}: ${err.message}`); + log(`๐Ÿ’พ Progress saved. You can restart to resume from ${formatShortDate(currentStart.toISOString())}`); break; } - if (start < new Date('2002-01-01')) { - log(`๐Ÿ›‘ Reached earliest supported CVE publication date โ€” halting backfill.`); + // Check if we've reached today + if (currentStart >= now) { + log(`๐ŸŽ‰ Reached current date โ€” backfill complete!`); break; } } @@ -291,9 +302,8 @@ async function importCVEFeedBackfill() { -//importCVEFeed().catch((err) => { -importCVEFeedBackfill(9000) // ~25 years (goes back to 2000) -.catch((err) => { +// Use importCVEFeed() for daily sync or importCVEFeedBackfill() for full backfill +importCVEFeedBackfill().catch((err) => { log(`โŒ Fatal error during import: ${err.message}`); logFile.end(); }); diff --git a/scripts/verifyCVECount.js b/scripts/verifyCVECount.js new file mode 100755 index 0000000..7412737 --- /dev/null +++ b/scripts/verifyCVECount.js @@ -0,0 +1,198 @@ +#!/usr/bin/env node + +import fs from 'fs'; +import { execSync } from 'child_process'; +import mysql from 'mysql2/promise'; +import path from 'path'; + +const REPO_URL = 'https://github.com/CVEProject/cvelistV5.git'; +const REPO_DIR = './cvelistV5'; + +function log(msg) { + const now = new Date().toLocaleString('en-AU', { + day: '2-digit', month: 'short', year: 'numeric', + hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: true + }).replace(/\b(AM|PM)\b/, m => m.toLowerCase()); + + const line = `[${now}] ${msg}`; + console.log(line); +} + +async function cloneOrPullRepo() { + log('๐Ÿ“ฆ Checking CVE repository...'); + + if (fs.existsSync(REPO_DIR)) { + log('๐Ÿ”„ Repository exists, pulling latest changes...'); + try { + execSync('git pull', { cwd: REPO_DIR, stdio: 'inherit' }); + log('โœ… Repository updated'); + } catch (err) { + log(`โš ๏ธ Git pull failed, trying fresh clone: ${err.message}`); + execSync(`rm -rf ${REPO_DIR}`); + execSync(`git clone --depth 1 ${REPO_URL} ${REPO_DIR}`, { stdio: 'inherit' }); + } + } else { + log('๐Ÿ“ฅ Cloning CVE repository (this may take a while)...'); + execSync(`git clone --depth 1 ${REPO_URL} ${REPO_DIR}`, { stdio: 'inherit' }); + log('โœ… Repository cloned'); + } +} + +function countCVEsInRepo() { + log('๐Ÿ” Counting CVEs in repository...'); + + const cvesDir = path.join(REPO_DIR, 'cves'); + let totalCount = 0; + const yearCounts = {}; + + if (!fs.existsSync(cvesDir)) { + log('โŒ CVEs directory not found'); + return { total: 0, byYear: {} }; + } + + // Iterate through year directories (e.g., cves/2023/) + const years = fs.readdirSync(cvesDir).filter(f => /^\d{4}$/.test(f)); + + for (const year of years) { + const yearPath = path.join(cvesDir, year); + if (!fs.statSync(yearPath).isDirectory()) continue; + + let yearCount = 0; + + // Each year has subdirectories like 0xxx, 1xxx, etc. + const subdirs = fs.readdirSync(yearPath); + + for (const subdir of subdirs) { + const subdirPath = path.join(yearPath, subdir); + if (!fs.statSync(subdirPath).isDirectory()) continue; + + // Count .json files in this subdirectory + const files = fs.readdirSync(subdirPath).filter(f => f.endsWith('.json')); + yearCount += files.length; + } + + totalCount += yearCount; + yearCounts[year] = yearCount; + } + + log(`๐Ÿ“Š Found ${totalCount} CVE JSON files in repository`); + return { total: totalCount, byYear: yearCounts }; +} + +async function countCVEsInDatabase() { + log('๐Ÿ—„๏ธ Counting CVEs in database...'); + + const db = await mysql.createConnection({ + host: process.env.DB_HOST, + port: process.env.DB_PORT || 3306, + user: process.env.DB_USER, + password: process.env.DB_PASSWORD, + database: process.env.DB_NAME, + }); + + // Total count + const [totalRows] = await db.query('SELECT COUNT(*) as total FROM cves'); + const total = totalRows[0].total; + + // Count by year + const [yearRows] = await db.query(` + SELECT + YEAR(published_date) as year, + COUNT(*) as count + FROM cves + WHERE published_date IS NOT NULL + GROUP BY YEAR(published_date) + ORDER BY year + `); + + const byYear = {}; + yearRows.forEach(row => { + byYear[row.year] = row.count; + }); + + await db.end(); + + log(`๐Ÿ“Š Found ${total} CVEs in database`); + return { total, byYear }; +} + +function compareResults(repo, db) { + log('\n๐Ÿ“‹ Comparison Report:'); + log('โ”'.repeat(60)); + log(`Repository Total: ${repo.total.toLocaleString()}`); + log(`Database Total: ${db.total.toLocaleString()}`); + log(`Difference: ${(db.total - repo.total).toLocaleString()}`); + log('โ”'.repeat(60)); + + // Get all years from both sources + const allYears = new Set([ + ...Object.keys(repo.byYear), + ...Object.keys(db.byYear) + ]); + + const sortedYears = Array.from(allYears).sort(); + + log('\n๐Ÿ“… Year-by-Year Breakdown:'); + log('โ”'.repeat(60)); + log('Year | Repository | Database | Difference'); + log('โ”'.repeat(60)); + + const missingYears = []; + + for (const year of sortedYears) { + const repoCount = repo.byYear[year] || 0; + const dbCount = db.byYear[year] || 0; + const diff = dbCount - repoCount; + + const diffStr = diff >= 0 ? `+${diff}` : diff.toString(); + log(`${year} | ${repoCount.toString().padStart(10)} | ${dbCount.toString().padStart(9)} | ${diffStr}`); + + if (Math.abs(diff) > 100) { + missingYears.push({ year, repoCount, dbCount, diff }); + } + } + + log('โ”'.repeat(60)); + + if (missingYears.length > 0) { + log('\nโš ๏ธ Years with significant differences (>100):'); + missingYears.forEach(({ year, repoCount, dbCount, diff }) => { + log(` ${year}: ${diff > 0 ? 'Extra' : 'Missing'} ${Math.abs(diff)} CVEs`); + }); + } + + const percentComplete = ((db.total / repo.total) * 100).toFixed(2); + log(`\nโœ… Database is ${percentComplete}% complete`); + + if (db.total >= repo.total) { + log('๐ŸŽ‰ Your database has all CVEs from the official repository!'); + } else { + log(`โš ๏ธ Missing ${(repo.total - db.total).toLocaleString()} CVEs`); + } +} + +async function main() { + try { + log('๐Ÿš€ Starting CVE verification...\n'); + + // Step 1: Clone or pull repo + await cloneOrPullRepo(); + + // Step 2: Count CVEs in repo + const repoStats = countCVEsInRepo(); + + // Step 3: Count CVEs in database + const dbStats = await countCVEsInDatabase(); + + // Step 4: Compare + compareResults(repoStats, dbStats); + + log('\nโœ… Verification complete!'); + } catch (err) { + log(`โŒ Error: ${err.message}`); + console.error(err); + process.exit(1); + } +} + +main(); diff --git a/scripts/verifyCVECountAPI.js b/scripts/verifyCVECountAPI.js new file mode 100644 index 0000000..56322ea --- /dev/null +++ b/scripts/verifyCVECountAPI.js @@ -0,0 +1,209 @@ +#!/usr/bin/env node + +import axios from 'axios'; +import mysql from 'mysql2/promise'; + +function log(msg) { + const now = new Date().toLocaleString('en-AU', { + day: '2-digit', month: 'short', year: 'numeric', + hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: true + }).replace(/\b(AM|PM)\b/, m => m.toLowerCase()); + + const line = `[${now}] ${msg}`; + console.log(line); +} + +async function getGitHubCVEStats() { + log('๐Ÿ“ก Fetching CVE statistics from GitHub API...'); + + try { + // Get repository tree (cves directory structure) + const response = await axios.get( + 'https://api.github.com/repos/CVEProject/cvelistV5/git/trees/main?recursive=1', + { + headers: { + 'Accept': 'application/vnd.github.v3+json', + 'User-Agent': 'CVE-Verification-Script' + } + } + ); + + const tree = response.data.tree; + + // Count .json files in cves/ directory + const cveFiles = tree.filter(item => + item.path.startsWith('cves/') && + item.path.endsWith('.json') && + item.type === 'blob' + ); + + // Group by year + const byYear = {}; + cveFiles.forEach(file => { + const match = file.path.match(/cves\/(\d{4})\//); + if (match) { + const year = match[1]; + byYear[year] = (byYear[year] || 0) + 1; + } + }); + + log(`โœ… Found ${cveFiles.length} CVE files in GitHub repository`); + + return { + total: cveFiles.length, + byYear, + lastCommit: response.data.sha + }; + } catch (err) { + log(`โŒ GitHub API error: ${err.message}`); + if (err.response?.status === 403) { + log('โš ๏ธ GitHub API rate limit exceeded. Try again later or clone the repository.'); + } + throw err; + } +} + +async function countCVEsInDatabase() { + log('๐Ÿ—„๏ธ Counting CVEs in database...'); + + const db = await mysql.createConnection({ + host: process.env.DB_HOST, + port: process.env.DB_PORT || 3306, + user: process.env.DB_USER, + password: process.env.DB_PASSWORD, + database: process.env.DB_NAME, + }); + + // Total count + const [totalRows] = await db.query('SELECT COUNT(*) as total FROM cves'); + const total = totalRows[0].total; + + // Count by year + const [yearRows] = await db.query(` + SELECT + YEAR(published_date) as year, + COUNT(*) as count + FROM cves + WHERE published_date IS NOT NULL + GROUP BY YEAR(published_date) + ORDER BY year + `); + + const byYear = {}; + yearRows.forEach(row => { + byYear[row.year] = row.count; + }); + + // Get date range + const [rangeRows] = await db.query(` + SELECT + MIN(published_date) as earliest, + MAX(published_date) as latest + FROM cves + `); + + await db.end(); + + log(`โœ… Found ${total} CVEs in database`); + return { + total, + byYear, + earliest: rangeRows[0].earliest, + latest: rangeRows[0].latest + }; +} + +function compareResults(github, db) { + log('\n๐Ÿ“‹ Comparison Report:'); + log('โ”'.repeat(70)); + log(`GitHub Repository: ${github.total.toLocaleString()} CVEs`); + log(`Your Database: ${db.total.toLocaleString()} CVEs`); + log(`Difference: ${(db.total - github.total).toLocaleString()}`); + log('โ”'.repeat(70)); + + log(`\nDatabase Date Range:`); + log(` Earliest: ${db.earliest ? new Date(db.earliest).toLocaleDateString() : 'N/A'}`); + log(` Latest: ${db.latest ? new Date(db.latest).toLocaleDateString() : 'N/A'}`); + + // Get all years + const allYears = new Set([ + ...Object.keys(github.byYear), + ...Object.keys(db.byYear) + ]); + + const sortedYears = Array.from(allYears).sort(); + + log('\n๐Ÿ“… Year-by-Year Breakdown:'); + log('โ”'.repeat(70)); + log('Year | GitHub | Database | Difference | % Complete'); + log('โ”'.repeat(70)); + + const significantDiffs = []; + + for (const year of sortedYears) { + const githubCount = github.byYear[year] || 0; + const dbCount = db.byYear[year] || 0; + const diff = dbCount - githubCount; + const pctComplete = githubCount > 0 ? ((dbCount / githubCount) * 100).toFixed(1) : '0.0'; + + const diffStr = diff >= 0 ? `+${diff}` : diff.toString(); + log(`${year} | ${githubCount.toString().padStart(10)} | ${dbCount.toString().padStart(10)} | ${diffStr.padStart(10)} | ${pctComplete.padStart(6)}%`); + + if (Math.abs(diff) > 100) { + significantDiffs.push({ year, githubCount, dbCount, diff }); + } + } + + log('โ”'.repeat(70)); + + if (significantDiffs.length > 0) { + log('\nโš ๏ธ Years with significant differences (>100):'); + significantDiffs.forEach(({ year, githubCount, dbCount, diff }) => { + if (diff < 0) { + log(` ${year}: Missing ${Math.abs(diff)} CVEs (${dbCount}/${githubCount})`); + } else { + log(` ${year}: Extra ${diff} CVEs (database has more than GitHub)`); + } + }); + } + + const percentComplete = github.total > 0 + ? ((db.total / github.total) * 100).toFixed(2) + : '0.00'; + + log(`\n๐Ÿ“Š Overall Completion: ${percentComplete}%`); + + if (db.total >= github.total) { + log('๐ŸŽ‰ Your database has all CVEs from the official GitHub repository!'); + if (db.total > github.total) { + log(' (You may have older CVEs or modified entries not in the current repository)'); + } + } else { + const missing = github.total - db.total; + log(`โš ๏ธ Missing ${missing.toLocaleString()} CVEs from GitHub repository`); + log(` Run the backfill script to sync missing CVEs.`); + } +} + +async function main() { + try { + log('๐Ÿš€ Starting CVE verification using GitHub API...\n'); + + // Step 1: Get stats from GitHub API + const githubStats = await getGitHubCVEStats(); + + // Step 2: Count CVEs in database + const dbStats = await countCVEsInDatabase(); + + // Step 3: Compare + compareResults(githubStats, dbStats); + + log('\nโœ… Verification complete!'); + } catch (err) { + log(`โŒ Error: ${err.message}`); + console.error(err); + process.exit(1); + } +} + +main(); diff --git a/src/main/java/com/psg/dlsysinfo/dl_sysinfo_server/controller/ScriptController.java b/src/main/java/com/psg/dlsysinfo/dl_sysinfo_server/controller/ScriptController.java index 40b38a3..4eb7fb5 100644 --- a/src/main/java/com/psg/dlsysinfo/dl_sysinfo_server/controller/ScriptController.java +++ b/src/main/java/com/psg/dlsysinfo/dl_sysinfo_server/controller/ScriptController.java @@ -84,6 +84,12 @@ public class ScriptController { return triggerScript("enrichCVE_MSRC.js", "๐Ÿ“ก MSRC sync launched in background.", getMsrcLogFile()); } + @PreAuthorize("hasRole('ADMIN')") + @PostMapping("/verify-cve-count") + public ResponseEntity verifyCveCount(@AuthenticationPrincipal Object user) { + return triggerScript("verifyCVECountAPI.js", "๐Ÿ” CVE verification started - comparing with GitHub API.", getCveLogFile()); + } + @PreAuthorize("hasRole('ADMIN')") @GetMapping("/fetch-cve/logs") public ResponseEntity fetchLogs(@AuthenticationPrincipal Object user) {