#!/usr/bin/env node import fs from 'fs'; import axios from 'axios'; import mysql from 'mysql2/promise'; const logFile = fs.createWriteStream('cve-sync.log', { flags: 'a', encoding: 'utf8', }); const RESUME_FILE = '.last_synced_date'; function saveLastSyncedDate(dateStr) { fs.writeFileSync(RESUME_FILE, dateStr); } function loadLastSyncedDate() { if (!fs.existsSync(RESUME_FILE)) return null; return fs.readFileSync(RESUME_FILE, 'utf8'); } function log(msg) { const now = new Date(); // Generate the locale string with hour12 enabled (e.g. "14 Apr 2025, 08:14:42 AM") const raw = now.toLocaleString('en-AU', { day: '2-digit', month: 'short', year: 'numeric', hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: true, }); // Regex to convert only the AM/PM to lowercase const formatted = raw.replace(/\b(AM|PM)\b/, (match) => match.toLowerCase()); const timestamp = `[${formatted}]`; const line = `${timestamp} ${msg}`; console.log(line); logFile.write(`${line}\n`); } function formatDate(isoString) { if (!isoString) return null; const date = new Date(isoString); return date.toISOString().slice(0, 19).replace('T', ' '); } function formatShortDate(isoString) { return new Date(isoString).toLocaleDateString('en-AU', { day: '2-digit', month: 'short', year: 'numeric', }).replace(/\b([A-Z])([a-z]+)\b/, (_, a, b) => a + b); // Capitalize only first letter of month } function extractCpeParts(cpe) { const parts = cpe.split(':'); return { vendor: parts[3] || null, product: parts[4] || null, version: parts[5] || null }; } function addDaysToISO(dateISO, days) { const date = new Date(dateISO); date.setDate(date.getDate() + days); return date.toISOString(); } const DB = await mysql.createConnection({ host: process.env.DB_HOST, port: process.env.DB_PORT || 3306, user: process.env.DB_USER, password: process.env.DB_PASSWORD, database: process.env.DB_NAME, }); const BASE_URL = 'https://services.nvd.nist.gov/rest/json/cves/2.0'; const API_KEY = process.env.NVD_API_KEY; const RESULTS_PER_PAGE = 2000; let MAX_RANGE_DAYS = Number(process.env.NVD_MAX_RANGE_DAYS || 120); log(`๐Ÿงช Getting CVEs from the last ${MAX_RANGE_DAYS} of days`); if (MAX_RANGE_DAYS > 120) { log("โš ๏ธ MAX_RANGE_DAYS exceeds NVD API limit. Defaulting to 120."); MAX_RANGE_DAYS = 120; } if (isNaN(MAX_RANGE_DAYS) || MAX_RANGE_DAYS < 1) { log("โš ๏ธ Invalid MAX_RANGE_DAYS. Using default of 7."); MAX_RANGE_DAYS = 7; } async function fetchCVEPage(startIndex, startDate, endDate) { try { const res = await axios.get(BASE_URL, { params: { pubStartDate: startDate, pubEndDate: endDate, startIndex, resultsPerPage: RESULTS_PER_PAGE, }, headers: API_KEY ? { apiKey: API_KEY } : {} }); return res.data; } catch (err) { log(`โŒ API error: ${err.response?.status} - ${err.response?.data?.message || err.message}`); throw err; } } async function processCVE(cveWrapper) { const cve = cveWrapper.cve; const cveId = cve.id; const title = cve.titles?.find(t => t.lang === 'en')?.title || ''; const desc = cve.descriptions.find(d => d.lang === 'en')?.value ?? ''; const published = formatDate(cve.published); const modified = formatDate(cve.lastModified); const metric = cve.metrics?.cvssMetricV31?.[0]; const severity = metric?.cvssData?.baseSeverity ?? null; const score = metric?.cvssData?.baseScore ?? null; const vector = metric?.cvssData?.vectorString ?? ''; try { await DB.execute( `INSERT INTO cves (id, title, description, published_date, last_modified_date, severity, cvss_score, cvss_vector, source) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE last_modified_date = VALUES(last_modified_date), severity = IFNULL(severity, VALUES(severity)), cvss_score = IFNULL(cvss_score, VALUES(cvss_score)), cvss_vector = IFNULL(cvss_vector, VALUES(cvss_vector)), title = IFNULL(title, VALUES(title)), source = IFNULL(source, VALUES(source))`, [cveId, title, desc, published, modified, severity, score, vector, 'NVD'] ); } catch (err) { log(`โŒ Error inserting CVE ${cveId}: ${err.message}`); } const configurations = cve.configurations ?? []; for (const node of configurations) { for (const match of node.nodes?.flatMap(n => n.cpeMatch ?? []) ?? []) { const cpe = match.criteria; const vulnerable = match.vulnerable ? 1 : 0; const start = match.versionStartIncluding || match.versionStartExcluding || null; const end = match.versionEndIncluding || match.versionEndExcluding || null; const { vendor, product, version } = extractCpeParts(cpe); try { await DB.execute( `INSERT INTO cpe_matches (cve_id, cpe_uri, version_start, version_end, vulnerable, vendor, product, version) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, [cveId, cpe, start, end, vulnerable, vendor, product, version] ); } catch (err) { log(`โš ๏ธ Error inserting CPE for CVE ${cveId}: ${err.message}`); } } } } async function getMostRecentModifiedDateFromDB() { const [rows] = await DB.query(`SELECT MAX(last_modified_date) AS lastMod FROM cves`); const lastMod = rows[0]?.lastMod; return lastMod ? new Date(lastMod).toISOString() : '2020-01-01T00:00:00.000Z'; } async function importCVEFeed() { const now = new Date(); const endDate = now.toISOString(); const startDateObj = new Date(now); startDateObj.setDate(startDateObj.getDate() - MAX_RANGE_DAYS); const startDate = startDateObj.toISOString(); log(`๐Ÿš€ CVE sync started`); log(`๐Ÿ”„ Initializing script...`); log(`๐Ÿ“ Launching script`); log(`๐Ÿ“… Starting CVE sync from ${startDate} to ${endDate}`); const humanStart = formatShortDate(startDate); const humanEnd = formatShortDate(endDate); log(`๐Ÿ“ก Fetching modified CVEs from ${humanStart} to ${humanEnd}...`); let startIndex = 0; let totalResults = Infinity; let pageCount = 0; do { const data = await fetchCVEPage(startIndex, startDate, endDate); const vulnerabilities = data.vulnerabilities || []; totalResults = data.totalResults ?? vulnerabilities.length; if (vulnerabilities.length === 0) { log(`โš ๏ธ No CVEs returned at index ${startIndex}`); break; } log(`๐Ÿ“„ Page ${++pageCount} โ€” Processing ${vulnerabilities.length} CVEs from index ${startIndex} of ~${totalResults}`); for (const vuln of vulnerabilities) { await processCVE(vuln); } startIndex += RESULTS_PER_PAGE; await new Promise((r) => setTimeout(r, 6000)); } while (startIndex < totalResults); log('โœ… CVE import complete!'); await DB.end(); logFile.end(); } async function importCVEFeedBackfill() { const now = new Date(); const resumeFrom = loadLastSyncedDate(); let startFrom = resumeFrom ? new Date(resumeFrom) : now; const MAX_RANGE_DAYS = 120; log(resumeFrom ? `๐Ÿ” Resuming CVE backfill from ${formatShortDate(startFrom.toISOString())}` : `โฎ๏ธ Starting CVE backfill from today (${formatShortDate(startFrom.toISOString())})` ); while (true) { const end = new Date(startFrom); const start = new Date(startFrom); start.setDate(start.getDate() - MAX_RANGE_DAYS + 1); // 120-day window const startISO = start.toISOString(); const endISO = end.toISOString(); const humanRange = `${formatShortDate(startISO)} to ${formatShortDate(endISO)}`; log(`๐Ÿ“ก Fetching published CVEs from ${humanRange}...`); let startIndex = 0; let totalResults = Infinity; let pageCount = 0; try { do { const data = await fetchCVEPage(startIndex, startISO, endISO); const vulnerabilities = data.vulnerabilities || []; totalResults = data.totalResults ?? vulnerabilities.length; if (vulnerabilities.length === 0) { log(`โš ๏ธ No CVEs returned for ${humanRange} at index ${startIndex}`); break; } log(`๐Ÿ“„ Page ${++pageCount} โ€” ${vulnerabilities.length} CVEs from index ${startIndex}`); for (const vuln of vulnerabilities) { await processCVE(vuln); } startIndex += RESULTS_PER_PAGE; await new Promise((r) => setTimeout(r, 6000)); } while (startIndex < totalResults); // Move the window backward saveLastSyncedDate(start.toISOString()); startFrom = start; } catch (err) { log(`โŒ Error during ${humanRange}: ${err.message}`); break; } if (start < new Date('2002-01-01')) { log(`๐Ÿ›‘ Reached earliest supported CVE publication date โ€” halting backfill.`); break; } } log('โœ… CVE backfill complete!'); await DB.end(); logFile.end(); } //importCVEFeed().catch((err) => { importCVEFeedBackfill(9000) // ~25 years (goes back to 2000) .catch((err) => { log(`โŒ Fatal error during import: ${err.message}`); logFile.end(); });