New Github support for CVE verification.

2025-10-08 10:41:19 +08:00
parent 80c6aae9c2
commit c43b3a65c5
8 changed files with 476 additions and 20 deletions
--- a/scripts/.env.test
+++ b/scripts/.env.test
@@ -0,0 +1,5 @@
+DB_HOST=db.psg.net.au
+DB_PORT=3307
+DB_USER=svc_sysinfo
+DB_PASSWORD=2pT08pEuxqFiN6eD348vBlgoMfyfOjGB
+DB_NAME=db_ld-spring-backend
--- a/scripts/.last_synced_date
+++ b/scripts/.last_synced_date
@@ -1 +0,0 @@
-2001-08-20T00:00:00.000Z
--- a/scripts/cve-sync.log
+++ b/scripts/cve-sync.log
@@ -98,3 +98,31 @@
 [08 Oct 2025, 09:22:22 am] ✅ CVE import complete!
 [08 Oct 2025, 09:22:22 am] ✅ CVE import complete!
 [08 Oct 2025, 01:22:22 am] ✅ fetchCVE.js finished with exit code: 0
+[08 Oct 2025, 02:21:34 am] 🚀 📡 CVE backfill launched - will sync back to 2002.
+[08 Oct 2025, 10:21:35 am] 🧪 Getting CVEs from the last 30 of days
+[08 Oct 2025, 10:21:35 am] 🧪 Getting CVEs from the last 30 of days
+[08 Oct 2025, 10:21:35 am] 🔁 Resuming CVE backfill from 20 Aug 2001
+[08 Oct 2025, 10:21:35 am] 📡 Fetching published CVEs from 23 Apr 2001 to 20 Aug 2001...
+[08 Oct 2025, 10:21:35 am] 🔁 Resuming CVE backfill from 20 Aug 2001
+[08 Oct 2025, 10:21:35 am] 📡 Fetching published CVEs from 23 Apr 2001 to 20 Aug 2001...
+[08 Oct 2025, 10:21:36 am] 📄 Page 1 — 594 CVEs from index 0
+[08 Oct 2025, 10:21:36 am] 📄 Page 1 — 594 CVEs from index 0
+[08 Oct 2025, 10:21:49 am] 🛑 Reached earliest supported CVE publication date — halting backfill.
+[08 Oct 2025, 10:21:49 am] ✅ CVE backfill complete!
+[08 Oct 2025, 10:21:49 am] 🛑 Reached earliest supported CVE publication date — halting backfill.
+[08 Oct 2025, 10:21:49 am] ✅ CVE backfill complete!
+[08 Oct 2025, 02:21:49 am] ✅ fetchCVE_withMORE.js finished with exit code: 0
+[08 Oct 2025, 02:22:45 am] 🚀 📡 CVE backfill launched - will sync back to 2002.
+[08 Oct 2025, 10:22:45 am] 🧪 Getting CVEs from the last 30 of days
+[08 Oct 2025, 10:22:45 am] 🧪 Getting CVEs from the last 30 of days
+[08 Oct 2025, 10:22:45 am] 🔁 Resuming CVE backfill from 23 Apr 2001
+[08 Oct 2025, 10:22:45 am] 📡 Fetching published CVEs from 25 Dec 2000 to 23 Apr 2001...
+[08 Oct 2025, 10:22:45 am] 🔁 Resuming CVE backfill from 23 Apr 2001
+[08 Oct 2025, 10:22:45 am] 📡 Fetching published CVEs from 25 Dec 2000 to 23 Apr 2001...
+[08 Oct 2025, 10:22:47 am] 📄 Page 1 — 389 CVEs from index 0
+[08 Oct 2025, 10:22:47 am] 📄 Page 1 — 389 CVEs from index 0
+[08 Oct 2025, 10:22:57 am] 🛑 Reached earliest supported CVE publication date — halting backfill.
+[08 Oct 2025, 10:22:57 am] ✅ CVE backfill complete!
+[08 Oct 2025, 10:22:57 am] 🛑 Reached earliest supported CVE publication date — halting backfill.
+[08 Oct 2025, 10:22:57 am] ✅ CVE backfill complete!
+[08 Oct 2025, 02:22:57 am] ✅ fetchCVE_withMORE.js finished with exit code: 0
--- a/scripts/cvelistV5
+++ b/scripts/cvelistV5
--- a/scripts/fetchCVE_withMORE.js
+++ b/scripts/fetchCVE_withMORE.js
@@ -223,21 +223,28 @@ async function importCVEFeed() {
 }

 async function importCVEFeedBackfill() {
-  const now = new Date();
-  const resumeFrom = loadLastSyncedDate();
-  let startFrom = resumeFrom ? new Date(resumeFrom) : now;
-
+  const EARLIEST_CVE_DATE = new Date('2002-01-01T00:00:00.000Z');
  const MAX_RANGE_DAYS = 120;

+  const resumeFrom = loadLastSyncedDate();
+  let currentStart = resumeFrom ? new Date(resumeFrom) : EARLIEST_CVE_DATE;
+
  log(resumeFrom
-    ? `🔁 Resuming CVE backfill from ${formatShortDate(startFrom.toISOString())}`
-    : `⏮️ Starting CVE backfill from today (${formatShortDate(startFrom.toISOString())})`
+    ? `🔁 Resuming CVE backfill from ${formatShortDate(currentStart.toISOString())}`
+    : `⏮️ Starting CVE backfill from ${formatShortDate(EARLIEST_CVE_DATE.toISOString())}`
  );

-  while (true) {
-    const end = new Date(startFrom);
-    const start = new Date(startFrom);
-    start.setDate(start.getDate() - MAX_RANGE_DAYS + 1); // 120-day window
+  const now = new Date();
+
+  while (currentStart < now) {
+    const start = new Date(currentStart);
+    const end = new Date(currentStart);
+    end.setDate(end.getDate() + MAX_RANGE_DAYS - 1); // 120-day window
+
+    // Don't go past today
+    if (end > now) {
+      end.setTime(now.getTime());
+    }

    const startISO = start.toISOString();
    const endISO = end.toISOString();
@@ -260,7 +267,7 @@ async function importCVEFeedBackfill() {
          break;
        }

-        log(`📄 Page ${++pageCount} — ${vulnerabilities.length} CVEs from index ${startIndex}`);
+        log(`📄 Page ${++pageCount} — ${vulnerabilities.length} CVEs from index ${startIndex} of ~${totalResults}`);

        for (const vuln of vulnerabilities) {
          await processCVE(vuln);
@@ -270,16 +277,20 @@ async function importCVEFeedBackfill() {
        await new Promise((r) => setTimeout(r, 6000));
      } while (startIndex < totalResults);

-      // Move the window backward
-      saveLastSyncedDate(start.toISOString());
-      startFrom = start;
+      // Move the window forward
+      currentStart.setDate(currentStart.getDate() + MAX_RANGE_DAYS);
+      saveLastSyncedDate(currentStart.toISOString());
+
+      log(`✅ Completed ${humanRange}. Next start: ${formatShortDate(currentStart.toISOString())}`);
    } catch (err) {
      log(`❌ Error during ${humanRange}: ${err.message}`);
+      log(`💾 Progress saved. You can restart to resume from ${formatShortDate(currentStart.toISOString())}`);
      break;
    }

-    if (start < new Date('2002-01-01')) {
-      log(`🛑 Reached earliest supported CVE publication date — halting backfill.`);
+    // Check if we've reached today
+    if (currentStart >= now) {
+      log(`🎉 Reached current date — backfill complete!`);
      break;
    }
  }
@@ -291,9 +302,8 @@ async function importCVEFeedBackfill() {



-//importCVEFeed().catch((err) => {
-importCVEFeedBackfill(9000) // ~25 years (goes back to 2000)
-.catch((err) => {
+// Use importCVEFeed() for daily sync or importCVEFeedBackfill() for full backfill
+importCVEFeedBackfill().catch((err) => {
  log(`❌ Fatal error during import: ${err.message}`);
  logFile.end();
 });
--- a/scripts/verifyCVECount.js
+++ b/scripts/verifyCVECount.js
@@ -0,0 +1,198 @@
+#!/usr/bin/env node
+
+import fs from 'fs';
+import { execSync } from 'child_process';
+import mysql from 'mysql2/promise';
+import path from 'path';
+
+const REPO_URL = 'https://github.com/CVEProject/cvelistV5.git';
+const REPO_DIR = './cvelistV5';
+
+function log(msg) {
+  const now = new Date().toLocaleString('en-AU', {
+    day: '2-digit', month: 'short', year: 'numeric',
+    hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: true
+  }).replace(/\b(AM|PM)\b/, m => m.toLowerCase());
+
+  const line = `[${now}] ${msg}`;
+  console.log(line);
+}
+
+async function cloneOrPullRepo() {
+  log('📦 Checking CVE repository...');
+
+  if (fs.existsSync(REPO_DIR)) {
+    log('🔄 Repository exists, pulling latest changes...');
+    try {
+      execSync('git pull', { cwd: REPO_DIR, stdio: 'inherit' });
+      log('✅ Repository updated');
+    } catch (err) {
+      log(`⚠️ Git pull failed, trying fresh clone: ${err.message}`);
+      execSync(`rm -rf ${REPO_DIR}`);
+      execSync(`git clone --depth 1 ${REPO_URL} ${REPO_DIR}`, { stdio: 'inherit' });
+    }
+  } else {
+    log('📥 Cloning CVE repository (this may take a while)...');
+    execSync(`git clone --depth 1 ${REPO_URL} ${REPO_DIR}`, { stdio: 'inherit' });
+    log('✅ Repository cloned');
+  }
+}
+
+function countCVEsInRepo() {
+  log('🔍 Counting CVEs in repository...');
+
+  const cvesDir = path.join(REPO_DIR, 'cves');
+  let totalCount = 0;
+  const yearCounts = {};
+
+  if (!fs.existsSync(cvesDir)) {
+    log('❌ CVEs directory not found');
+    return { total: 0, byYear: {} };
+  }
+
+  // Iterate through year directories (e.g., cves/2023/)
+  const years = fs.readdirSync(cvesDir).filter(f => /^\d{4}$/.test(f));
+
+  for (const year of years) {
+    const yearPath = path.join(cvesDir, year);
+    if (!fs.statSync(yearPath).isDirectory()) continue;
+
+    let yearCount = 0;
+
+    // Each year has subdirectories like 0xxx, 1xxx, etc.
+    const subdirs = fs.readdirSync(yearPath);
+
+    for (const subdir of subdirs) {
+      const subdirPath = path.join(yearPath, subdir);
+      if (!fs.statSync(subdirPath).isDirectory()) continue;
+
+      // Count .json files in this subdirectory
+      const files = fs.readdirSync(subdirPath).filter(f => f.endsWith('.json'));
+      yearCount += files.length;
+    }
+
+    totalCount += yearCount;
+    yearCounts[year] = yearCount;
+  }
+
+  log(`📊 Found ${totalCount} CVE JSON files in repository`);
+  return { total: totalCount, byYear: yearCounts };
+}
+
+async function countCVEsInDatabase() {
+  log('🗄️ Counting CVEs in database...');
+
+  const db = await mysql.createConnection({
+    host: process.env.DB_HOST,
+    port: process.env.DB_PORT || 3306,
+    user: process.env.DB_USER,
+    password: process.env.DB_PASSWORD,
+    database: process.env.DB_NAME,
+  });
+
+  // Total count
+  const [totalRows] = await db.query('SELECT COUNT(*) as total FROM cves');
+  const total = totalRows[0].total;
+
+  // Count by year
+  const [yearRows] = await db.query(`
+    SELECT
+      YEAR(published_date) as year,
+      COUNT(*) as count
+    FROM cves
+    WHERE published_date IS NOT NULL
+    GROUP BY YEAR(published_date)
+    ORDER BY year
+  `);
+
+  const byYear = {};
+  yearRows.forEach(row => {
+    byYear[row.year] = row.count;
+  });
+
+  await db.end();
+
+  log(`📊 Found ${total} CVEs in database`);
+  return { total, byYear };
+}
+
+function compareResults(repo, db) {
+  log('\n📋 Comparison Report:');
+  log('━'.repeat(60));
+  log(`Repository Total:  ${repo.total.toLocaleString()}`);
+  log(`Database Total:    ${db.total.toLocaleString()}`);
+  log(`Difference:        ${(db.total - repo.total).toLocaleString()}`);
+  log('━'.repeat(60));
+
+  // Get all years from both sources
+  const allYears = new Set([
+    ...Object.keys(repo.byYear),
+    ...Object.keys(db.byYear)
+  ]);
+
+  const sortedYears = Array.from(allYears).sort();
+
+  log('\n📅 Year-by-Year Breakdown:');
+  log('━'.repeat(60));
+  log('Year   | Repository | Database  | Difference');
+  log('━'.repeat(60));
+
+  const missingYears = [];
+
+  for (const year of sortedYears) {
+    const repoCount = repo.byYear[year] || 0;
+    const dbCount = db.byYear[year] || 0;
+    const diff = dbCount - repoCount;
+
+    const diffStr = diff >= 0 ? `+${diff}` : diff.toString();
+    log(`${year}  | ${repoCount.toString().padStart(10)} | ${dbCount.toString().padStart(9)} | ${diffStr}`);
+
+    if (Math.abs(diff) > 100) {
+      missingYears.push({ year, repoCount, dbCount, diff });
+    }
+  }
+
+  log('━'.repeat(60));
+
+  if (missingYears.length > 0) {
+    log('\n⚠️ Years with significant differences (>100):');
+    missingYears.forEach(({ year, repoCount, dbCount, diff }) => {
+      log(`   ${year}: ${diff > 0 ? 'Extra' : 'Missing'} ${Math.abs(diff)} CVEs`);
+    });
+  }
+
+  const percentComplete = ((db.total / repo.total) * 100).toFixed(2);
+  log(`\n✅ Database is ${percentComplete}% complete`);
+
+  if (db.total >= repo.total) {
+    log('🎉 Your database has all CVEs from the official repository!');
+  } else {
+    log(`⚠️ Missing ${(repo.total - db.total).toLocaleString()} CVEs`);
+  }
+}
+
+async function main() {
+  try {
+    log('🚀 Starting CVE verification...\n');
+
+    // Step 1: Clone or pull repo
+    await cloneOrPullRepo();
+
+    // Step 2: Count CVEs in repo
+    const repoStats = countCVEsInRepo();
+
+    // Step 3: Count CVEs in database
+    const dbStats = await countCVEsInDatabase();
+
+    // Step 4: Compare
+    compareResults(repoStats, dbStats);
+
+    log('\n✅ Verification complete!');
+  } catch (err) {
+    log(`❌ Error: ${err.message}`);
+    console.error(err);
+    process.exit(1);
+  }
+}
+
+main();
--- a/scripts/verifyCVECountAPI.js
+++ b/scripts/verifyCVECountAPI.js
@@ -0,0 +1,209 @@
+#!/usr/bin/env node
+
+import axios from 'axios';
+import mysql from 'mysql2/promise';
+
+function log(msg) {
+  const now = new Date().toLocaleString('en-AU', {
+    day: '2-digit', month: 'short', year: 'numeric',
+    hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: true
+  }).replace(/\b(AM|PM)\b/, m => m.toLowerCase());
+
+  const line = `[${now}] ${msg}`;
+  console.log(line);
+}
+
+async function getGitHubCVEStats() {
+  log('📡 Fetching CVE statistics from GitHub API...');
+
+  try {
+    // Get repository tree (cves directory structure)
+    const response = await axios.get(
+      'https://api.github.com/repos/CVEProject/cvelistV5/git/trees/main?recursive=1',
+      {
+        headers: {
+          'Accept': 'application/vnd.github.v3+json',
+          'User-Agent': 'CVE-Verification-Script'
+        }
+      }
+    );
+
+    const tree = response.data.tree;
+
+    // Count .json files in cves/ directory
+    const cveFiles = tree.filter(item =>
+      item.path.startsWith('cves/') &&
+      item.path.endsWith('.json') &&
+      item.type === 'blob'
+    );
+
+    // Group by year
+    const byYear = {};
+    cveFiles.forEach(file => {
+      const match = file.path.match(/cves\/(\d{4})\//);
+      if (match) {
+        const year = match[1];
+        byYear[year] = (byYear[year] || 0) + 1;
+      }
+    });
+
+    log(`✅ Found ${cveFiles.length} CVE files in GitHub repository`);
+
+    return {
+      total: cveFiles.length,
+      byYear,
+      lastCommit: response.data.sha
+    };
+  } catch (err) {
+    log(`❌ GitHub API error: ${err.message}`);
+    if (err.response?.status === 403) {
+      log('⚠️ GitHub API rate limit exceeded. Try again later or clone the repository.');
+    }
+    throw err;
+  }
+}
+
+async function countCVEsInDatabase() {
+  log('🗄️ Counting CVEs in database...');
+
+  const db = await mysql.createConnection({
+    host: process.env.DB_HOST,
+    port: process.env.DB_PORT || 3306,
+    user: process.env.DB_USER,
+    password: process.env.DB_PASSWORD,
+    database: process.env.DB_NAME,
+  });
+
+  // Total count
+  const [totalRows] = await db.query('SELECT COUNT(*) as total FROM cves');
+  const total = totalRows[0].total;
+
+  // Count by year
+  const [yearRows] = await db.query(`
+    SELECT
+      YEAR(published_date) as year,
+      COUNT(*) as count
+    FROM cves
+    WHERE published_date IS NOT NULL
+    GROUP BY YEAR(published_date)
+    ORDER BY year
+  `);
+
+  const byYear = {};
+  yearRows.forEach(row => {
+    byYear[row.year] = row.count;
+  });
+
+  // Get date range
+  const [rangeRows] = await db.query(`
+    SELECT
+      MIN(published_date) as earliest,
+      MAX(published_date) as latest
+    FROM cves
+  `);
+
+  await db.end();
+
+  log(`✅ Found ${total} CVEs in database`);
+  return {
+    total,
+    byYear,
+    earliest: rangeRows[0].earliest,
+    latest: rangeRows[0].latest
+  };
+}
+
+function compareResults(github, db) {
+  log('\n📋 Comparison Report:');
+  log('━'.repeat(70));
+  log(`GitHub Repository: ${github.total.toLocaleString()} CVEs`);
+  log(`Your Database:     ${db.total.toLocaleString()} CVEs`);
+  log(`Difference:        ${(db.total - github.total).toLocaleString()}`);
+  log('━'.repeat(70));
+
+  log(`\nDatabase Date Range:`);
+  log(`  Earliest: ${db.earliest ? new Date(db.earliest).toLocaleDateString() : 'N/A'}`);
+  log(`  Latest:   ${db.latest ? new Date(db.latest).toLocaleDateString() : 'N/A'}`);
+
+  // Get all years
+  const allYears = new Set([
+    ...Object.keys(github.byYear),
+    ...Object.keys(db.byYear)
+  ]);
+
+  const sortedYears = Array.from(allYears).sort();
+
+  log('\n📅 Year-by-Year Breakdown:');
+  log('━'.repeat(70));
+  log('Year   | GitHub     | Database   | Difference | % Complete');
+  log('━'.repeat(70));
+
+  const significantDiffs = [];
+
+  for (const year of sortedYears) {
+    const githubCount = github.byYear[year] || 0;
+    const dbCount = db.byYear[year] || 0;
+    const diff = dbCount - githubCount;
+    const pctComplete = githubCount > 0 ? ((dbCount / githubCount) * 100).toFixed(1) : '0.0';
+
+    const diffStr = diff >= 0 ? `+${diff}` : diff.toString();
+    log(`${year}  | ${githubCount.toString().padStart(10)} | ${dbCount.toString().padStart(10)} | ${diffStr.padStart(10)} | ${pctComplete.padStart(6)}%`);
+
+    if (Math.abs(diff) > 100) {
+      significantDiffs.push({ year, githubCount, dbCount, diff });
+    }
+  }
+
+  log('━'.repeat(70));
+
+  if (significantDiffs.length > 0) {
+    log('\n⚠️ Years with significant differences (>100):');
+    significantDiffs.forEach(({ year, githubCount, dbCount, diff }) => {
+      if (diff < 0) {
+        log(`   ${year}: Missing ${Math.abs(diff)} CVEs (${dbCount}/${githubCount})`);
+      } else {
+        log(`   ${year}: Extra ${diff} CVEs (database has more than GitHub)`);
+      }
+    });
+  }
+
+  const percentComplete = github.total > 0
+    ? ((db.total / github.total) * 100).toFixed(2)
+    : '0.00';
+
+  log(`\n📊 Overall Completion: ${percentComplete}%`);
+
+  if (db.total >= github.total) {
+    log('🎉 Your database has all CVEs from the official GitHub repository!');
+    if (db.total > github.total) {
+      log('   (You may have older CVEs or modified entries not in the current repository)');
+    }
+  } else {
+    const missing = github.total - db.total;
+    log(`⚠️ Missing ${missing.toLocaleString()} CVEs from GitHub repository`);
+    log(`   Run the backfill script to sync missing CVEs.`);
+  }
+}
+
+async function main() {
+  try {
+    log('🚀 Starting CVE verification using GitHub API...\n');
+
+    // Step 1: Get stats from GitHub API
+    const githubStats = await getGitHubCVEStats();
+
+    // Step 2: Count CVEs in database
+    const dbStats = await countCVEsInDatabase();
+
+    // Step 3: Compare
+    compareResults(githubStats, dbStats);
+
+    log('\n✅ Verification complete!');
+  } catch (err) {
+    log(`❌ Error: ${err.message}`);
+    console.error(err);
+    process.exit(1);
+  }
+}
+
+main();
--- a/src/main/java/com/psg/dlsysinfo/dl_sysinfo_server/controller/ScriptController.java
+++ b/src/main/java/com/psg/dlsysinfo/dl_sysinfo_server/controller/ScriptController.java
@@ -84,6 +84,12 @@ public class ScriptController {
        return triggerScript("enrichCVE_MSRC.js", "📡 MSRC sync launched in background.", getMsrcLogFile());
    }

+    @PreAuthorize("hasRole('ADMIN')")
+    @PostMapping("/verify-cve-count")
+    public ResponseEntity<String> verifyCveCount(@AuthenticationPrincipal Object user) {
+        return triggerScript("verifyCVECountAPI.js", "🔍 CVE verification started - comparing with GitHub API.", getCveLogFile());
+    }
+
    @PreAuthorize("hasRole('ADMIN')")
    @GetMapping("/fetch-cve/logs")
    public ResponseEntity<String> fetchLogs(@AuthenticationPrincipal Object user) {