299 lines
9.1 KiB
JavaScript
299 lines
9.1 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import fs from 'fs';
|
|
import axios from 'axios';
|
|
import mysql from 'mysql2/promise';
|
|
|
|
const logFile = fs.createWriteStream('cve-sync.log', {
|
|
flags: 'a',
|
|
encoding: 'utf8',
|
|
});
|
|
|
|
const RESUME_FILE = '.last_synced_date';
|
|
function saveLastSyncedDate(dateStr) {
|
|
fs.writeFileSync(RESUME_FILE, dateStr);
|
|
}
|
|
|
|
function loadLastSyncedDate() {
|
|
if (!fs.existsSync(RESUME_FILE)) return null;
|
|
return fs.readFileSync(RESUME_FILE, 'utf8');
|
|
}
|
|
|
|
|
|
function log(msg) {
|
|
const now = new Date();
|
|
|
|
// Generate the locale string with hour12 enabled (e.g. "14 Apr 2025, 08:14:42 AM")
|
|
const raw = now.toLocaleString('en-AU', {
|
|
day: '2-digit',
|
|
month: 'short',
|
|
year: 'numeric',
|
|
hour: '2-digit',
|
|
minute: '2-digit',
|
|
second: '2-digit',
|
|
hour12: true,
|
|
});
|
|
|
|
// Regex to convert only the AM/PM to lowercase
|
|
const formatted = raw.replace(/\b(AM|PM)\b/, (match) => match.toLowerCase());
|
|
|
|
const timestamp = `[${formatted}]`;
|
|
const line = `${timestamp} ${msg}`;
|
|
console.log(line);
|
|
logFile.write(`${line}\n`);
|
|
}
|
|
|
|
|
|
|
|
function formatDate(isoString) {
|
|
if (!isoString) return null;
|
|
const date = new Date(isoString);
|
|
return date.toISOString().slice(0, 19).replace('T', ' ');
|
|
}
|
|
|
|
function formatShortDate(isoString) {
|
|
return new Date(isoString).toLocaleDateString('en-AU', {
|
|
day: '2-digit',
|
|
month: 'short',
|
|
year: 'numeric',
|
|
}).replace(/\b([A-Z])([a-z]+)\b/, (_, a, b) => a + b); // Capitalize only first letter of month
|
|
}
|
|
|
|
|
|
function extractCpeParts(cpe) {
|
|
const parts = cpe.split(':');
|
|
return {
|
|
vendor: parts[3] || null,
|
|
product: parts[4] || null,
|
|
version: parts[5] || null
|
|
};
|
|
}
|
|
|
|
function addDaysToISO(dateISO, days) {
|
|
const date = new Date(dateISO);
|
|
date.setDate(date.getDate() + days);
|
|
return date.toISOString();
|
|
}
|
|
|
|
const DB = await mysql.createConnection({
|
|
host: process.env.DB_HOST,
|
|
user: process.env.DB_USER,
|
|
password: process.env.DB_PASSWORD,
|
|
database: process.env.DB_NAME,
|
|
});
|
|
|
|
const BASE_URL = 'https://services.nvd.nist.gov/rest/json/cves/2.0';
|
|
const API_KEY = process.env.NVD_API_KEY;
|
|
const RESULTS_PER_PAGE = 2000;
|
|
let MAX_RANGE_DAYS = Number(process.env.NVD_MAX_RANGE_DAYS || 120);
|
|
|
|
log(`🧪 Getting CVEs from the last ${MAX_RANGE_DAYS} of days`);
|
|
|
|
if (MAX_RANGE_DAYS > 120) {
|
|
log("⚠️ MAX_RANGE_DAYS exceeds NVD API limit. Defaulting to 120.");
|
|
MAX_RANGE_DAYS = 120;
|
|
}
|
|
if (isNaN(MAX_RANGE_DAYS) || MAX_RANGE_DAYS < 1) {
|
|
log("⚠️ Invalid MAX_RANGE_DAYS. Using default of 7.");
|
|
MAX_RANGE_DAYS = 7;
|
|
}
|
|
|
|
async function fetchCVEPage(startIndex, startDate, endDate) {
|
|
try {
|
|
const res = await axios.get(BASE_URL, {
|
|
params: {
|
|
pubStartDate: startDate,
|
|
pubEndDate: endDate,
|
|
startIndex,
|
|
resultsPerPage: RESULTS_PER_PAGE,
|
|
},
|
|
headers: API_KEY ? { apiKey: API_KEY } : {}
|
|
});
|
|
|
|
return res.data;
|
|
} catch (err) {
|
|
log(`❌ API error: ${err.response?.status} - ${err.response?.data?.message || err.message}`);
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
async function processCVE(cveWrapper) {
|
|
const cve = cveWrapper.cve;
|
|
const cveId = cve.id;
|
|
const title = cve.titles?.find(t => t.lang === 'en')?.title || '';
|
|
const desc = cve.descriptions.find(d => d.lang === 'en')?.value ?? '';
|
|
const published = formatDate(cve.published);
|
|
const modified = formatDate(cve.lastModified);
|
|
const metric = cve.metrics?.cvssMetricV31?.[0];
|
|
const severity = metric?.cvssData?.baseSeverity ?? null;
|
|
const score = metric?.cvssData?.baseScore ?? null;
|
|
const vector = metric?.cvssData?.vectorString ?? '';
|
|
|
|
try {
|
|
await DB.execute(
|
|
`INSERT INTO cves (id, title, description, published_date, last_modified_date, severity, cvss_score, cvss_vector, source)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON DUPLICATE KEY UPDATE
|
|
last_modified_date = VALUES(last_modified_date),
|
|
severity = IFNULL(severity, VALUES(severity)),
|
|
cvss_score = IFNULL(cvss_score, VALUES(cvss_score)),
|
|
cvss_vector = IFNULL(cvss_vector, VALUES(cvss_vector)),
|
|
title = IFNULL(title, VALUES(title)),
|
|
source = IFNULL(source, VALUES(source))`,
|
|
[cveId, title, desc, published, modified, severity, score, vector, 'NVD']
|
|
);
|
|
} catch (err) {
|
|
log(`❌ Error inserting CVE ${cveId}: ${err.message}`);
|
|
}
|
|
|
|
const configurations = cve.configurations ?? [];
|
|
for (const node of configurations) {
|
|
for (const match of node.nodes?.flatMap(n => n.cpeMatch ?? []) ?? []) {
|
|
const cpe = match.criteria;
|
|
const vulnerable = match.vulnerable ? 1 : 0;
|
|
const start = match.versionStartIncluding || match.versionStartExcluding || null;
|
|
const end = match.versionEndIncluding || match.versionEndExcluding || null;
|
|
|
|
const { vendor, product, version } = extractCpeParts(cpe);
|
|
|
|
try {
|
|
await DB.execute(
|
|
`INSERT INTO cpe_matches (cve_id, cpe_uri, version_start, version_end, vulnerable, vendor, product, version)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
[cveId, cpe, start, end, vulnerable, vendor, product, version]
|
|
);
|
|
} catch (err) {
|
|
log(`⚠️ Error inserting CPE for CVE ${cveId}: ${err.message}`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async function getMostRecentModifiedDateFromDB() {
|
|
const [rows] = await DB.query(`SELECT MAX(last_modified_date) AS lastMod FROM cves`);
|
|
const lastMod = rows[0]?.lastMod;
|
|
return lastMod ? new Date(lastMod).toISOString() : '2020-01-01T00:00:00.000Z';
|
|
}
|
|
|
|
async function importCVEFeed() {
|
|
const now = new Date();
|
|
const endDate = now.toISOString();
|
|
|
|
const startDateObj = new Date(now);
|
|
startDateObj.setDate(startDateObj.getDate() - MAX_RANGE_DAYS);
|
|
const startDate = startDateObj.toISOString();
|
|
|
|
log(`🚀 CVE sync started`);
|
|
log(`🔄 Initializing script...`);
|
|
log(`📍 Launching script`);
|
|
log(`📅 Starting CVE sync from ${startDate} to ${endDate}`);
|
|
|
|
const humanStart = formatShortDate(startDate);
|
|
const humanEnd = formatShortDate(endDate);
|
|
log(`📡 Fetching modified CVEs from ${humanStart} to ${humanEnd}...`);
|
|
|
|
let startIndex = 0;
|
|
let totalResults = Infinity;
|
|
let pageCount = 0;
|
|
|
|
do {
|
|
const data = await fetchCVEPage(startIndex, startDate, endDate);
|
|
const vulnerabilities = data.vulnerabilities || [];
|
|
totalResults = data.totalResults ?? vulnerabilities.length;
|
|
|
|
if (vulnerabilities.length === 0) {
|
|
log(`⚠️ No CVEs returned at index ${startIndex}`);
|
|
break;
|
|
}
|
|
|
|
log(`📄 Page ${++pageCount} — Processing ${vulnerabilities.length} CVEs from index ${startIndex} of ~${totalResults}`);
|
|
|
|
for (const vuln of vulnerabilities) {
|
|
await processCVE(vuln);
|
|
}
|
|
|
|
startIndex += RESULTS_PER_PAGE;
|
|
await new Promise((r) => setTimeout(r, 6000));
|
|
} while (startIndex < totalResults);
|
|
|
|
log('✅ CVE import complete!');
|
|
await DB.end();
|
|
logFile.end();
|
|
}
|
|
|
|
async function importCVEFeedBackfill() {
|
|
const now = new Date();
|
|
const resumeFrom = loadLastSyncedDate();
|
|
let startFrom = resumeFrom ? new Date(resumeFrom) : now;
|
|
|
|
const MAX_RANGE_DAYS = 120;
|
|
|
|
log(resumeFrom
|
|
? `🔁 Resuming CVE backfill from ${formatShortDate(startFrom.toISOString())}`
|
|
: `⏮️ Starting CVE backfill from today (${formatShortDate(startFrom.toISOString())})`
|
|
);
|
|
|
|
while (true) {
|
|
const end = new Date(startFrom);
|
|
const start = new Date(startFrom);
|
|
start.setDate(start.getDate() - MAX_RANGE_DAYS + 1); // 120-day window
|
|
|
|
const startISO = start.toISOString();
|
|
const endISO = end.toISOString();
|
|
const humanRange = `${formatShortDate(startISO)} to ${formatShortDate(endISO)}`;
|
|
|
|
log(`📡 Fetching published CVEs from ${humanRange}...`);
|
|
|
|
let startIndex = 0;
|
|
let totalResults = Infinity;
|
|
let pageCount = 0;
|
|
|
|
try {
|
|
do {
|
|
const data = await fetchCVEPage(startIndex, startISO, endISO);
|
|
const vulnerabilities = data.vulnerabilities || [];
|
|
totalResults = data.totalResults ?? vulnerabilities.length;
|
|
|
|
if (vulnerabilities.length === 0) {
|
|
log(`⚠️ No CVEs returned for ${humanRange} at index ${startIndex}`);
|
|
break;
|
|
}
|
|
|
|
log(`📄 Page ${++pageCount} — ${vulnerabilities.length} CVEs from index ${startIndex}`);
|
|
|
|
for (const vuln of vulnerabilities) {
|
|
await processCVE(vuln);
|
|
}
|
|
|
|
startIndex += RESULTS_PER_PAGE;
|
|
await new Promise((r) => setTimeout(r, 6000));
|
|
} while (startIndex < totalResults);
|
|
|
|
// Move the window backward
|
|
saveLastSyncedDate(start.toISOString());
|
|
startFrom = start;
|
|
} catch (err) {
|
|
log(`❌ Error during ${humanRange}: ${err.message}`);
|
|
break;
|
|
}
|
|
|
|
if (start < new Date('2002-01-01')) {
|
|
log(`🛑 Reached earliest supported CVE publication date — halting backfill.`);
|
|
break;
|
|
}
|
|
}
|
|
|
|
log('✅ CVE backfill complete!');
|
|
await DB.end();
|
|
logFile.end();
|
|
}
|
|
|
|
|
|
|
|
//importCVEFeed().catch((err) => {
|
|
importCVEFeedBackfill(9000) // ~25 years (goes back to 2000)
|
|
.catch((err) => {
|
|
log(`❌ Fatal error during import: ${err.message}`);
|
|
logFile.end();
|
|
});
|