sonderau/ld-sysinfo-server-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit of ld-sysinfo-server backend

Browse Source
This commit is contained in:
Bailey Taylor
2025-09-19 02:09:05 +00:00
commit c0349c106b
1760 changed files with 216243 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
scripts/.enrichment_resume Normal file
View File

@@ -0,0 +1 @@
CVE-1999-0057

7
scripts/.env.local Normal file
View File

@@ -0,0 +1,7 @@
DB_HOST=localhost
DB_USER=root
DB_PASSWORD=6DRR4xWvHBhSqLGtIOEKa7gHjKnX33Hf
DB_NAME=db_ld-spring-backend
NVD_API_KEY=42b4f093-e8c4-4110-a7d1-6ab2ba6234aa
NVD_MAX_RANGE_DAYS=30

1
scripts/.last_synced_date Normal file
View File

@@ -0,0 +1 @@
2001-08-20T00:00:00.000Z

48
scripts/cve-sync.log Normal file
View File

@@ -0,0 +1,48 @@
[27 May 2025, 12:53:07 pm] 🚀 📡 CVE sync launched in background.
[27 May 2025, 12:53:08 pm] 🧪 Getting CVEs from the last 7 of days
[27 May 2025, 12:53:08 pm] 🧪 Getting CVEs from the last 7 of days
[27 May 2025, 12:53:08 pm] 🚀 CVE sync started
[27 May 2025, 12:53:08 pm] 🔄 Initializing script...
[27 May 2025, 12:53:08 pm] 📍 Launching script
[27 May 2025, 12:53:08 pm] 📅 Starting CVE sync from 2025-05-20T04:53:08.063Z to 2025-05-27T04:53:08.063Z
[27 May 2025, 12:53:08 pm] 📡 Fetching modified CVEs from 20 May 2025 to 27 May 2025...
[27 May 2025, 12:53:08 pm] 🚀 CVE sync started
[27 May 2025, 12:53:08 pm] 🔄 Initializing script...
[27 May 2025, 12:53:08 pm] 📍 Launching script
[27 May 2025, 12:53:08 pm] 📅 Starting CVE sync from 2025-05-20T04:53:08.063Z to 2025-05-27T04:53:08.063Z
[27 May 2025, 12:53:08 pm] 📡 Fetching modified CVEs from 20 May 2025 to 27 May 2025...
[27 May 2025, 12:53:10 pm] 📄 Page 1 — Processing 2000 CVEs from index 0 of ~2212
[27 May 2025, 12:53:10 pm] 📄 Page 1 — Processing 2000 CVEs from index 0 of ~2212
[27 May 2025, 12:53:27 pm] 📄 Page 2 — Processing 212 CVEs from index 2000 of ~2212
[27 May 2025, 12:53:27 pm] 📄 Page 2 — Processing 212 CVEs from index 2000 of ~2212
[27 May 2025, 12:53:33 pm] ✅ CVE import complete!
[27 May 2025, 12:53:33 pm] ✅ CVE import complete!
[27 May 2025, 12:53:33 pm] ✅ fetchCVE.js finished with exit code: 0
[27 May 2025, 12:54:46 pm] 🚀 📡 CVE sync launched in background.
[27 May 2025, 12:54:46 pm] 🧪 Getting CVEs from the last 30 of days
[27 May 2025, 12:54:46 pm] 🧪 Getting CVEs from the last 30 of days
[27 May 2025, 12:54:46 pm] 🚀 CVE sync started
[27 May 2025, 12:54:46 pm] 🔄 Initializing script...
[27 May 2025, 12:54:46 pm] 📍 Launching script
[27 May 2025, 12:54:46 pm] 📅 Starting CVE sync from 2025-04-27T04:54:46.513Z to 2025-05-27T04:54:46.513Z
[27 May 2025, 12:54:46 pm] 📡 Fetching modified CVEs from 27 Apr 2025 to 27 May 2025...
[27 May 2025, 12:54:46 pm] 🚀 CVE sync started
[27 May 2025, 12:54:46 pm] 🔄 Initializing script...
[27 May 2025, 12:54:46 pm] 📍 Launching script
[27 May 2025, 12:54:46 pm] 📅 Starting CVE sync from 2025-04-27T04:54:46.513Z to 2025-05-27T04:54:46.513Z
[27 May 2025, 12:54:46 pm] 📡 Fetching modified CVEs from 27 Apr 2025 to 27 May 2025...
[27 May 2025, 12:54:49 pm] 📄 Page 1 — Processing 2000 CVEs from index 0 of ~10257
[27 May 2025, 12:54:49 pm] 📄 Page 1 — Processing 2000 CVEs from index 0 of ~10257
[27 May 2025, 12:55:55 pm] 📄 Page 2 — Processing 2000 CVEs from index 2000 of ~10257
[27 May 2025, 12:55:55 pm] 📄 Page 2 — Processing 2000 CVEs from index 2000 of ~10257
[27 May 2025, 12:56:18 pm] 📄 Page 3 — Processing 2000 CVEs from index 4000 of ~10257
[27 May 2025, 12:56:18 pm] 📄 Page 3 — Processing 2000 CVEs from index 4000 of ~10257
[27 May 2025, 12:56:35 pm] 📄 Page 4 — Processing 2000 CVEs from index 6000 of ~10257
[27 May 2025, 12:56:35 pm] 📄 Page 4 — Processing 2000 CVEs from index 6000 of ~10257
[27 May 2025, 12:57:03 pm] 📄 Page 5 — Processing 2000 CVEs from index 8000 of ~10257
[27 May 2025, 12:57:03 pm] 📄 Page 5 — Processing 2000 CVEs from index 8000 of ~10257
[27 May 2025, 12:57:15 pm] 📄 Page 6 — Processing 257 CVEs from index 10000 of ~10257
[27 May 2025, 12:57:15 pm] 📄 Page 6 — Processing 257 CVEs from index 10000 of ~10257
[27 May 2025, 12:57:21 pm] ✅ CVE import complete!
[27 May 2025, 12:57:21 pm] ✅ CVE import complete!
[27 May 2025, 12:57:21 pm] ✅ fetchCVE.js finished with exit code: 0

177
scripts/enrichCVE_MSRC.js Normal file
View File

@@ -0,0 +1,177 @@
#!/usr/bin/env node
import axios from 'axios';
import mysql from 'mysql2/promise';
import dotenv from 'dotenv';
dotenv.config({ path: '.env.local' });
const DB = await mysql.createConnection({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
});
function formatDate(isoString) {
if (!isoString) return null;
const date = new Date(isoString);
return date.toISOString().slice(0, 19).replace('T', ' ');
}
function getLocalizedText(entry, fallback = '') {
if (!entry || typeof entry !== 'object') return fallback;
// MSRC: sometimes the object directly has "Value"
if ('Value' in entry && typeof entry.Value === 'string') {
return entry.Value;
}
// If it's localized (e.g., { "en-US": { Value: ... } })
if (entry['en-US']?.Value) return entry['en-US'].Value;
// Fallback: first valid value
for (const key in entry) {
if (entry[key]?.Value) return entry[key].Value;
}
return fallback;
}
function extractDescription(notes = [], fallback = '') {
const note = notes.find(n =>
n.Title === 'Description' &&
n.Type === 2 &&
typeof n.Value === 'string' &&
n.Value.trim().length > 0
);
return note?.Value || fallback;
}
function extractAffectedProducts(vuln) {
const fixed = vuln.ProductStatuses?.Fixed ?? [];
const known = vuln.ProductStatuses?.KnownAffected ?? [];
const underInvestigation = vuln.ProductStatuses?.UnderInvestigation ?? [];
return [...new Set([...fixed, ...known, ...underInvestigation])].join(', ');
}
async function fetchCVRFDoc(cvrfId) {
const url = `https://api.msrc.microsoft.com/cvrf/v3.0/cvrf/${cvrfId}`;
try {
const res = await axios.get(url, {
headers: { Accept: 'application/json' },
});
return res.data;
} catch (err) {
console.error(`❌ Failed to fetch CVRF for ${cvrfId}:`, err.message);
return null;
}
}
async function enrichCVE(cve, data) {
const [rows] = await DB.execute(
`SELECT title, severity, description, cvss_score, cvss_vector FROM cves WHERE id = ?`,
[cve]
);
if (rows.length === 0) return;
const existing = rows[0];
const updateFields = {
title: existing.title || data.title,
severity: existing.severity || data.severity,
description: (existing.description?.length ?? 0) < 20 ? data.description : existing.description,
cvss_score: existing.cvss_score ?? data.cvssScore,
cvss_vector: existing.cvss_vector || data.cvssVector,
};
await DB.execute(
`UPDATE cves
SET title = ?, severity = ?, description = ?, cvss_score = ?, cvss_vector = ?
WHERE id = ?`,
[updateFields.title, updateFields.severity, updateFields.description, updateFields.cvss_score, updateFields.cvss_vector, cve]
);
}
async function storeVulnerability(v, published, modified) {
const cve = v.CVE;
console.log(`⏳ Processing ${cve}...`);
const title = getLocalizedText(v.Title, '');
console.debug(`🧪 Raw Title for ${cve}:`, JSON.stringify(v.Title, null, 2));
console.debug(`➡️ Parsed Title: "${title}"`);
const description = extractDescription(v.Notes, title);
const affectedProducts = extractAffectedProducts(v);
const severity = v.Threats?.find(t => t.Type === 'Severity')?.Description?.Value || '';
const exploitability = v.Threats?.find(t => t.Type === 'Exploitability')?.Description?.Value || '';
const cvssSet = v.CVSSScoreSets?.[0];
const cvssScore = cvssSet?.BaseScore || null;
const cvssVector = cvssSet?.Vector || '';
const cweList = Array.isArray(v.CWE) ? v.CWE : [];
await DB.execute(
`INSERT INTO microsoft_cves (cve_id, title, severity, published_date, last_modified_date,
affected_products, description, exploitability, cvss_score, cvss_vector)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
last_modified_date = VALUES(last_modified_date),
affected_products = VALUES(affected_products),
severity = VALUES(severity),
description = VALUES(description),
exploitability = VALUES(exploitability),
cvss_score = VALUES(cvss_score),
cvss_vector = VALUES(cvss_vector)`,
[cve, title, severity, published, modified, affectedProducts, description, exploitability, cvssScore, cvssVector]
);
for (const { ID: cweId, Value: cweName } of cweList) {
if (!cweId) continue;
await DB.execute(
`INSERT IGNORE INTO cwe_entries (cwe_id, cwe_name) VALUES (?, ?)`,
[cweId, cweName || null]
);
await DB.execute(
`INSERT IGNORE INTO cve_cwe (cve_id, cwe_id) VALUES (?, ?)`,
[cve, cweId]
);
}
await enrichCVE(cve, { title, severity, description, cvssScore, cvssVector });
console.log(`✅ Updated ${cve} in master table`);
}
function getPreviousMonthID() {
const now = new Date();
now.setMonth(now.getMonth() - 1);
const year = now.getFullYear();
const month = now.toLocaleString('en-US', { month: 'short' }); // e.g., "Mar"
return `${year}-${month}`;
}
async function enrichPreviousMonth() {
const cvrfId = getPreviousMonthID();
console.log(`📡 Fetching CVEs for ${cvrfId}...`);
const data = await fetchCVRFDoc(cvrfId);
if (!data) return;
const published = formatDate(data.DocumentTracking?.InitialReleaseDate);
const modified = formatDate(data.DocumentTracking?.CurrentReleaseDate);
const vulns = data.Vulnerability || [];
console.log(`📋 Found ${vulns.length} vulnerabilities.`);
for (const v of vulns) {
await storeVulnerability(v, published, modified);
}
console.log(`🎉 Enrichment complete for ${cvrfId}`);
await DB.end();
}
console.log("🚀 Starting MSRC enrichment of CVEs...");
await enrichPreviousMonth();

208
scripts/fetchCVE.js Normal file
View File

@@ -0,0 +1,208 @@
#!/usr/bin/env node
import fs from 'fs';
import axios from 'axios';
import mysql from 'mysql2/promise';
const logFile = fs.createWriteStream('cve-sync.log', {
flags: 'a',
encoding: 'utf8',
});
function log(msg) {
const now = new Date();
// Generate the locale string with hour12 enabled (e.g. "14 Apr 2025, 08:14:42 AM")
const raw = now.toLocaleString('en-AU', {
day: '2-digit',
month: 'short',
year: 'numeric',
hour: '2-digit',
minute: '2-digit',
second: '2-digit',
hour12: true,
});
// Regex to convert only the AM/PM to lowercase
const formatted = raw.replace(/\b(AM|PM)\b/, (match) => match.toLowerCase());
const timestamp = `[${formatted}]`;
const line = `${timestamp} ${msg}`;
console.log(line);
logFile.write(`${line}\n`);
}
function formatDate(isoString) {
if (!isoString) return null;
const date = new Date(isoString);
return date.toISOString().slice(0, 19).replace('T', ' ');
}
function formatShortDate(isoString) {
return new Date(isoString).toLocaleDateString('en-AU', {
day: '2-digit',
month: 'short',
year: 'numeric',
}).replace(/\b([A-Z])([a-z]+)\b/, (_, a, b) => a + b); // Capitalize only first letter of month
}
function extractCpeParts(cpe) {
const parts = cpe.split(':');
return {
vendor: parts[3] || null,
product: parts[4] || null,
version: parts[5] || null
};
}
function addDaysToISO(dateISO, days) {
const date = new Date(dateISO);
date.setDate(date.getDate() + days);
return date.toISOString();
}
const DB = await mysql.createConnection({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
});
const BASE_URL = 'https://services.nvd.nist.gov/rest/json/cves/2.0';
const API_KEY = process.env.NVD_API_KEY;
const RESULTS_PER_PAGE = 2000;
let MAX_RANGE_DAYS = Number(process.env.NVD_MAX_RANGE_DAYS || 7);
log(`🧪 Getting CVEs from the last ${MAX_RANGE_DAYS} of days`);
if (MAX_RANGE_DAYS > 120) {
log("⚠️ MAX_RANGE_DAYS exceeds NVD API limit. Defaulting to 120.");
MAX_RANGE_DAYS = 120;
}
if (isNaN(MAX_RANGE_DAYS) || MAX_RANGE_DAYS < 1) {
log("⚠️ Invalid MAX_RANGE_DAYS. Using default of 7.");
MAX_RANGE_DAYS = 7;
}
async function fetchCVEPage(startIndex, startDate, endDate) {
try {
const res = await axios.get(BASE_URL, {
params: {
startIndex,
resultsPerPage: RESULTS_PER_PAGE,
lastModStartDate: startDate,
lastModEndDate: endDate,
},
headers: API_KEY ? { apiKey: API_KEY } : {}
});
return res.data;
} catch (err) {
log(`❌ API error: ${err.response?.status} - ${err.response?.data?.message || err.message}`);
throw err;
}
}
async function processCVE(cveWrapper) {
const cve = cveWrapper.cve;
const cveId = cve.id;
const desc = cve.descriptions.find(d => d.lang === 'en')?.value ?? '';
const published = formatDate(cve.published);
const modified = formatDate(cve.lastModified);
const severity = cve.metrics?.cvssMetricV31?.[0]?.cvssData?.baseSeverity ?? null;
const score = cve.metrics?.cvssMetricV31?.[0]?.cvssData?.baseScore ?? null;
try {
await DB.execute(
`INSERT INTO cves (id, description, published_date, last_modified_date, severity, cvss_score)
VALUES (?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE last_modified_date = VALUES(last_modified_date)`,
[cveId, desc, published, modified, severity, score]
);
} catch (err) {
log(`❌ Error inserting CVE ${cveId}: ${err.message}`);
}
const configurations = cve.configurations ?? [];
for (const node of configurations) {
for (const match of node.nodes?.flatMap(n => n.cpeMatch ?? []) ?? []) {
const cpe = match.criteria;
const vulnerable = match.vulnerable ? 1 : 0;
const start = match.versionStartIncluding || match.versionStartExcluding || null;
const end = match.versionEndIncluding || match.versionEndExcluding || null;
const { vendor, product, version } = extractCpeParts(cpe);
try {
await DB.execute(
`INSERT INTO cpe_matches (cve_id, cpe_uri, version_start, version_end, vulnerable, vendor, product, version)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
[cveId, cpe, start, end, vulnerable, vendor, product, version]
);
} catch (err) {
log(`⚠️ Error inserting CPE for CVE ${cveId}: ${err.message}`);
}
}
}
}
async function getMostRecentModifiedDateFromDB() {
const [rows] = await DB.query(`SELECT MAX(last_modified_date) AS lastMod FROM cves`);
const lastMod = rows[0]?.lastMod;
return lastMod ? new Date(lastMod).toISOString() : '2020-01-01T00:00:00.000Z';
}
async function importCVEFeed() {
const now = new Date();
const endDate = now.toISOString();
const startDateObj = new Date(now);
startDateObj.setDate(startDateObj.getDate() - MAX_RANGE_DAYS);
const startDate = startDateObj.toISOString();
log(`🚀 CVE sync started`);
log(`🔄 Initializing script...`);
log(`📍 Launching script`);
log(`📅 Starting CVE sync from ${startDate} to ${endDate}`);
const humanStart = formatShortDate(startDate);
const humanEnd = formatShortDate(endDate);
log(`📡 Fetching modified CVEs from ${humanStart} to ${humanEnd}...`);
let startIndex = 0;
let totalResults = Infinity;
let pageCount = 0;
do {
const data = await fetchCVEPage(startIndex, startDate, endDate);
const vulnerabilities = data.vulnerabilities || [];
totalResults = data.totalResults ?? vulnerabilities.length;
if (vulnerabilities.length === 0) {
log(`⚠️ No CVEs returned at index ${startIndex}`);
break;
}
log(`📄 Page ${++pageCount} — Processing ${vulnerabilities.length} CVEs from index ${startIndex} of ~${totalResults}`);
for (const vuln of vulnerabilities) {
await processCVE(vuln);
}
startIndex += RESULTS_PER_PAGE;
await new Promise((r) => setTimeout(r, 6000));
} while (startIndex < totalResults);
log('✅ CVE import complete!');
await DB.end();
logFile.end();
}
importCVEFeed().catch((err) => {
log(`❌ Fatal error during import: ${err.message}`);
logFile.end();
});

457
scripts/fetchCVE_v2.js Normal file
View File

@@ -0,0 +1,457 @@
#!/usr/bin/env node
import dotenv from 'dotenv';
dotenv.config({ path: '.env.local' });
import fs from 'fs';
import axios from 'axios';
import mysql from 'mysql2/promise';
const logFile = fs.createWriteStream('cve-sync.log', {
flags: 'a',
encoding: 'utf8',
});
const RESUME_FILE = '.enrichment_resume';
const DB = await mysql.createConnection({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
});
const BASE_URL = 'https://services.nvd.nist.gov/rest/json/cves/2.0';
const API_KEY = process.env.NVD_API_KEY;
const RESULTS_PER_PAGE = 2000;
let MAX_RANGE_DAYS = Number(process.env.NVD_MAX_RANGE_DAYS || 120);
log(`🧪 Getting CVEs from the last ${MAX_RANGE_DAYS} of days`);
if (MAX_RANGE_DAYS > 120) {
log("⚠️ MAX_RANGE_DAYS exceeds NVD API limit. Defaulting to 120.");
MAX_RANGE_DAYS = 120;
}
if (isNaN(MAX_RANGE_DAYS) || MAX_RANGE_DAYS < 1) {
log("⚠️ Invalid MAX_RANGE_DAYS. Using default of 7.");
MAX_RANGE_DAYS = 7;
}
function saveLastProcessedCVE(cveId) {
fs.writeFileSync(RESUME_FILE, cveId, 'utf8');
}
function loadLastProcessedCVE() {
if (!fs.existsSync(RESUME_FILE)) return null;
return fs.readFileSync(RESUME_FILE, 'utf8');
}
function log(msg) {
const now = new Date();
// Generate the locale string with hour12 enabled (e.g. "14 Apr 2025, 08:14:42 AM")
const raw = now.toLocaleString('en-AU', {
day: '2-digit',
month: 'short',
year: 'numeric',
hour: '2-digit',
minute: '2-digit',
second: '2-digit',
hour12: true,
});
// Regex to convert only the AM/PM to lowercase
const formatted = raw.replace(/\b(AM|PM)\b/, (match) => match.toLowerCase());
const timestamp = `[${formatted}]`;
const line = `${timestamp} ${msg}`;
console.log(line);
logFile.write(`${line}\n`);
}
function formatDate(isoString) {
if (!isoString) return null;
const date = new Date(isoString);
return date.toISOString().slice(0, 19).replace('T', ' ');
}
function formatShortDate(isoString) {
return new Date(isoString).toLocaleDateString('en-AU', {
day: '2-digit',
month: 'short',
year: 'numeric',
}).replace(/\b([A-Z])([a-z]+)\b/, (_, a, b) => a + b); // Capitalize only first letter of month
}
function extractCpeParts(cpe) {
const parts = cpe.split(':');
return {
vendor: parts[3] || null,
product: parts[4] || null,
version: parts[5] || null
};
}
function addDaysToISO(dateISO, days) {
const date = new Date(dateISO);
date.setDate(date.getDate() + days);
return date.toISOString();
}
async function fetchCVEPage(startIndex, startDate, endDate, extraOptions = {}) {
try {
const res = await axios.get(BASE_URL, {
params: {
pubStartDate: startDate,
pubEndDate: endDate,
startIndex,
resultsPerPage: RESULTS_PER_PAGE,
...extraOptions,
},
headers: API_KEY ? { apiKey: API_KEY } : {}
});
return res.data;
} catch (err) {
log(`❌ API error: ${err.response?.status} - ${err.response?.data?.message || err.message}`);
throw err;
}
}
async function processCVE(cveWrapper) {
const cve = cveWrapper.cve;
const cveId = cve.id;
const title = cve.titles?.find(t => t.lang === 'en')?.title || '';
const desc = cve.descriptions?.find(d => d.lang === 'en')?.value || '';
const published = formatDate(cve.published);
const modified = formatDate(cve.lastModified);
// CVSSv2
const metricV2 = cve.metrics?.cvssMetricV2?.[0];
const severityV2 = metricV2?.cvssData?.baseSeverity || null;
const scoreV2 = metricV2?.cvssData?.baseScore || null;
const vectorV2 = metricV2?.cvssData?.vectorString || '';
// CVSSv3
const metricV3 = cve.metrics?.cvssMetricV31?.[0];
const severityV3 = metricV3?.cvssData?.baseSeverity || null;
const scoreV3 = metricV3?.cvssData?.baseScore || null;
const vectorV3 = metricV3?.cvssData?.vectorString || '';
// CVSSv4
const metricV4 = cve.metrics?.cvssMetricV40?.[0] || cve.metrics?.cvssMetricV4?.[0];
const severityV4 = metricV4?.cvssData?.baseSeverity || null;
const scoreV4 = metricV4?.cvssData?.baseScore || null;
const vectorV4 = metricV4?.cvssData?.vectorString || '';
// CWE IDs
const cweIds = (cve.weaknesses || [])
.flatMap(w => w.description || [])
.filter(desc => desc.lang === 'en')
.map(desc => desc.value)
.join(',');
// References
const references = (cve.references || [])
.map(ref => ref.url)
.join(',');
// Tags
const cveTags = cve.cveMetadata?.cveTags || [];
const hasKev = cveTags.includes('Known_Exploited_Vulnerability');
const hasCertNotes = cveTags.includes('CERT-VN');
const hasCertAlerts = cveTags.includes('US-CERT-TA');
try {
await DB.execute(
`INSERT INTO cves (
id, title, description, published_date, last_modified_date,
severity_v2, cvss_score_v2, cvss_vector_v2,
severity_v3, cvss_score_v3, cvss_vector_v3,
severity_v4, cvss_score_v4, cvss_vector_v4,
cwe_ids, \`references\`, hasKev, hasCertNotes, hasCertAlerts, source
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
last_modified_date = VALUES(last_modified_date),
severity_v2 = IFNULL(severity_v2, VALUES(severity_v2)),
cvss_score_v2 = IFNULL(cvss_score_v2, VALUES(cvss_score_v2)),
cvss_vector_v2 = IFNULL(cvss_vector_v2, VALUES(cvss_vector_v2)),
severity_v3 = IFNULL(severity_v3, VALUES(severity_v3)),
cvss_score_v3 = IFNULL(cvss_score_v3, VALUES(cvss_score_v3)),
cvss_vector_v3 = IFNULL(cvss_vector_v3, VALUES(cvss_vector_v3)),
severity_v4 = IFNULL(severity_v4, VALUES(severity_v4)),
cvss_score_v4 = IFNULL(cvss_score_v4, VALUES(cvss_score_v4)),
cvss_vector_v4 = IFNULL(cvss_vector_v4, VALUES(cvss_vector_v4)),
cwe_ids = IFNULL(cwe_ids, VALUES(cwe_ids)),
\`references\` = IFNULL(\`references\`, VALUES(\`references\`)),
hasKev = VALUES(hasKev),
hasCertNotes = VALUES(hasCertNotes),
hasCertAlerts = VALUES(hasCertAlerts),
source = VALUES(source)
`,
[
cveId, title, desc, published, modified,
severityV2, scoreV2, vectorV2,
severityV3, scoreV3, vectorV3,
severityV4, scoreV4, vectorV4,
cweIds, references, hasKev ? 1 : 0, hasCertNotes ? 1 : 0, hasCertAlerts ? 1 : 0, 'NVD'
]
);
} catch (err) {
log(`❌ Error inserting CVE ${cveId}: ${err.message}`);
}
}
async function getMostRecentModifiedDateFromDB() {
const [rows] = await DB.query(`SELECT MAX(last_modified_date) AS lastMod FROM cves`);
const lastMod = rows[0]?.lastMod;
return lastMod ? new Date(lastMod).toISOString() : '2020-01-01T00:00:00.000Z';
}
async function importCVEFeed() {
const now = new Date();
const endDate = now.toISOString();
const startDateObj = new Date(now);
startDateObj.setDate(startDateObj.getDate() - MAX_RANGE_DAYS);
const startDate = startDateObj.toISOString();
log(`🚀 CVE sync started`);
log(`🔄 Initializing script...`);
log(`📍 Launching script`);
log(`📅 Starting CVE sync from ${startDate} to ${endDate}`);
const humanStart = formatShortDate(startDate);
const humanEnd = formatShortDate(endDate);
log(`📡 Fetching modified CVEs from ${humanStart} to ${humanEnd}...`);
let startIndex = 0;
let totalResults = Infinity;
let pageCount = 0;
do {
const data = await fetchCVEPage(startIndex, startDate, endDate);
const vulnerabilities = data.vulnerabilities || [];
totalResults = data.totalResults ?? vulnerabilities.length;
if (vulnerabilities.length === 0) {
log(`⚠️ No CVEs returned at index ${startIndex}`);
break;
}
log(`📄 Page ${++pageCount} — Processing ${vulnerabilities.length} CVEs from index ${startIndex} of ~${totalResults}`);
for (const vuln of vulnerabilities) {
await processCVE(vuln);
}
startIndex += RESULTS_PER_PAGE;
await new Promise((r) => setTimeout(r, 6000));
} while (startIndex < totalResults);
log('✅ CVE import complete!');
await DB.end();
logFile.end();
}
async function importCVEFeedBackfill() {
const now = new Date();
const resumeFrom = loadLastSyncedDate();
let startFrom = resumeFrom ? new Date(resumeFrom) : now;
const MAX_RANGE_DAYS = 120;
log(resumeFrom
? `🔁 Resuming CVE backfill from ${formatShortDate(startFrom.toISOString())}`
: `⏮️ Starting CVE backfill from today (${formatShortDate(startFrom.toISOString())})`
);
while (true) {
const end = new Date(startFrom);
const start = new Date(startFrom);
start.setDate(start.getDate() - MAX_RANGE_DAYS + 1); // 120-day window
const startISO = start.toISOString();
const endISO = end.toISOString();
const humanRange = `${formatShortDate(startISO)} to ${formatShortDate(endISO)}`;
log(`📡 Fetching published CVEs from ${humanRange}...`);
let startIndex = 0;
let totalResults = Infinity;
let pageCount = 0;
try {
do {
const data = await fetchCVEPage(startIndex, startISO, endISO);
const vulnerabilities = data.vulnerabilities || [];
totalResults = data.totalResults ?? vulnerabilities.length;
if (vulnerabilities.length === 0) {
log(`⚠️ No CVEs returned for ${humanRange} at index ${startIndex}`);
break;
}
log(`📄 Page ${++pageCount} — ${vulnerabilities.length} CVEs from index ${startIndex}`);
for (const vuln of vulnerabilities) {
await processCVE(vuln);
}
startIndex += RESULTS_PER_PAGE;
await new Promise((r) => setTimeout(r, 6000));
} while (startIndex < totalResults);
// Move the window backward
saveLastSyncedDate(start.toISOString());
startFrom = start;
} catch (err) {
log(`❌ Error during ${humanRange}: ${err.message}`);
break;
}
if (start < new Date('2002-01-01')) {
log(`🛑 Reached earliest supported CVE publication date — halting backfill.`);
break;
}
}
log('✅ CVE backfill complete!');
await DB.end();
logFile.end();
}
async function importCVEEnrichmentFromDB() {
log(`🔍 Starting CVE enrichment from existing database records...`);
const lastProcessed = loadLastProcessedCVE();
if (lastProcessed) {
log(`🔁 Resuming enrichment from after ${lastProcessed}`);
}
const [rows] = await DB.query(`
SELECT id
FROM cves
WHERE severity_v3 IS NULL OR cvss_score_v3 IS NULL OR cvss_vector_v3 IS NULL
ORDER BY id ASC
`);
if (rows.length === 0) {
log(`✅ All CVEs seem enriched! Nothing to do.`);
await DB.end();
logFile.end();
return;
}
let startIndex = 0;
if (lastProcessed) {
const resumeIndex = rows.findIndex(r => r.id === lastProcessed);
if (resumeIndex !== -1 && resumeIndex + 1 < rows.length) {
startIndex = resumeIndex + 1; // Start *after* last processed
}
}
log(`📄 Found ${rows.length} CVEs needing enrichment, starting at index ${startIndex}.`);
for (let i = startIndex; i < rows.length; i++) {
const row = rows[i];
const cveId = row.id;
try {
const res = await axios.get(BASE_URL, {
params: { cveId },
headers: API_KEY ? { apiKey: API_KEY } : {}
});
const vulnerabilities = res.data.vulnerabilities || [];
if (vulnerabilities.length > 0) {
await processCVE(vulnerabilities[0]);
log(`✅ Enriched ${cveId}`);
} else {
log(`⚠️ No data found for ${cveId}`);
}
saveLastProcessedCVE(cveId); // ✨ Save after each successful CVE
await new Promise(r => setTimeout(r, 6000)); // Sleep to respect rate limit
} catch (err) {
log(`❌ Failed to fetch/enrich ${cveId}: ${err.message}`);
}
}
log('✅ CVE enrichment batch complete!');
fs.unlinkSync(RESUME_FILE); // ✨ Cleanup resume file if done
await DB.end();
logFile.end();
}
async function importCVEEnrichmentFast() {
log(`🚀 Starting FAST CVE enrichment by modified date...`);
const now = new Date();
const earliestDate = new Date('2002-01-01T00:00:00.000Z');
let endDateObj = new Date(); // most recent
const STEP_DAYS = 120;
while (endDateObj > earliestDate) {
const startDateObj = new Date(endDateObj);
startDateObj.setDate(startDateObj.getDate() - STEP_DAYS);
const startISO = startDateObj.toISOString();
const endISO = endDateObj.toISOString();
const humanRange = `${formatShortDate(startISO)} to ${formatShortDate(endISO)}`;
log(`📅 Fetching modified CVEs from ${humanRange}`);
let startIndex = 0;
let totalResults = 0;
let processedCount = 0;
let pageCount = 0;
try {
const initial = await fetchCVEPage(0, startISO, endISO);
totalResults = initial.totalResults || 0;
if (totalResults === 0) {
log(`⚠️ No CVEs to process in this window.`);
endDateObj = startDateObj;
continue;
}
log(`📦 Found ${totalResults} CVEs to enrich from ${humanRange}`);
while (startIndex < totalResults) {
const data = startIndex === 0 ? initial : await fetchCVEPage(startIndex, startISO, endISO);
const vulnerabilities = data.vulnerabilities || [];
log(`📄 Page ${++pageCount} — ${vulnerabilities.length} CVEs (Index ${startIndex})`);
for (const vuln of vulnerabilities) {
await processCVE(vuln);
processedCount++;
if (processedCount % 100 === 0 || processedCount === totalResults) {
const pct = ((processedCount / totalResults) * 100).toFixed(1);
log(`📊 Progress: ${processedCount}/${totalResults} CVEs (${pct}%)`);
}
}
startIndex += RESULTS_PER_PAGE;
await new Promise(r => setTimeout(r, 6000)); // API rate limit
}
} catch (err) {
log(`❌ Error during enrichment for ${humanRange}: ${err.message}`);
}
endDateObj = startDateObj; // 🕒 step backward
}
log('✅ Full enrichment pass complete!');
await DB.end();
logFile.end();
}
importCVEEnrichmentFast().catch((err) => {
log(`❌ Fatal error during enrichment: ${err.message}`);
logFile.end();
});

298
scripts/fetchCVE_withMORE.js Normal file
View File

@@ -0,0 +1,298 @@
#!/usr/bin/env node
import fs from 'fs';
import axios from 'axios';
import mysql from 'mysql2/promise';
const logFile = fs.createWriteStream('cve-sync.log', {
flags: 'a',
encoding: 'utf8',
});
const RESUME_FILE = '.last_synced_date';
function saveLastSyncedDate(dateStr) {
fs.writeFileSync(RESUME_FILE, dateStr);
}
function loadLastSyncedDate() {
if (!fs.existsSync(RESUME_FILE)) return null;
return fs.readFileSync(RESUME_FILE, 'utf8');
}
function log(msg) {
const now = new Date();
// Generate the locale string with hour12 enabled (e.g. "14 Apr 2025, 08:14:42 AM")
const raw = now.toLocaleString('en-AU', {
day: '2-digit',
month: 'short',
year: 'numeric',
hour: '2-digit',
minute: '2-digit',
second: '2-digit',
hour12: true,
});
// Regex to convert only the AM/PM to lowercase
const formatted = raw.replace(/\b(AM|PM)\b/, (match) => match.toLowerCase());
const timestamp = `[${formatted}]`;
const line = `${timestamp} ${msg}`;
console.log(line);
logFile.write(`${line}\n`);
}
function formatDate(isoString) {
if (!isoString) return null;
const date = new Date(isoString);
return date.toISOString().slice(0, 19).replace('T', ' ');
}
function formatShortDate(isoString) {
return new Date(isoString).toLocaleDateString('en-AU', {
day: '2-digit',
month: 'short',
year: 'numeric',
}).replace(/\b([A-Z])([a-z]+)\b/, (_, a, b) => a + b); // Capitalize only first letter of month
}
function extractCpeParts(cpe) {
const parts = cpe.split(':');
return {
vendor: parts[3] || null,
product: parts[4] || null,
version: parts[5] || null
};
}
function addDaysToISO(dateISO, days) {
const date = new Date(dateISO);
date.setDate(date.getDate() + days);
return date.toISOString();
}
const DB = await mysql.createConnection({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
});
const BASE_URL = 'https://services.nvd.nist.gov/rest/json/cves/2.0';
const API_KEY = process.env.NVD_API_KEY;
const RESULTS_PER_PAGE = 2000;
let MAX_RANGE_DAYS = Number(process.env.NVD_MAX_RANGE_DAYS || 120);
log(`🧪 Getting CVEs from the last ${MAX_RANGE_DAYS} of days`);
if (MAX_RANGE_DAYS > 120) {
log("⚠️ MAX_RANGE_DAYS exceeds NVD API limit. Defaulting to 120.");
MAX_RANGE_DAYS = 120;
}
if (isNaN(MAX_RANGE_DAYS) || MAX_RANGE_DAYS < 1) {
log("⚠️ Invalid MAX_RANGE_DAYS. Using default of 7.");
MAX_RANGE_DAYS = 7;
}
async function fetchCVEPage(startIndex, startDate, endDate) {
try {
const res = await axios.get(BASE_URL, {
params: {
pubStartDate: startDate,
pubEndDate: endDate,
startIndex,
resultsPerPage: RESULTS_PER_PAGE,
},
headers: API_KEY ? { apiKey: API_KEY } : {}
});
return res.data;
} catch (err) {
log(`❌ API error: ${err.response?.status} - ${err.response?.data?.message || err.message}`);
throw err;
}
}
async function processCVE(cveWrapper) {
const cve = cveWrapper.cve;
const cveId = cve.id;
const title = cve.titles?.find(t => t.lang === 'en')?.title || '';
const desc = cve.descriptions.find(d => d.lang === 'en')?.value ?? '';
const published = formatDate(cve.published);
const modified = formatDate(cve.lastModified);
const metric = cve.metrics?.cvssMetricV31?.[0];
const severity = metric?.cvssData?.baseSeverity ?? null;
const score = metric?.cvssData?.baseScore ?? null;
const vector = metric?.cvssData?.vectorString ?? '';
try {
await DB.execute(
`INSERT INTO cves (id, title, description, published_date, last_modified_date, severity, cvss_score, cvss_vector, source)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
last_modified_date = VALUES(last_modified_date),
severity = IFNULL(severity, VALUES(severity)),
cvss_score = IFNULL(cvss_score, VALUES(cvss_score)),
cvss_vector = IFNULL(cvss_vector, VALUES(cvss_vector)),
title = IFNULL(title, VALUES(title)),
source = IFNULL(source, VALUES(source))`,
[cveId, title, desc, published, modified, severity, score, vector, 'NVD']
);
} catch (err) {
log(`❌ Error inserting CVE ${cveId}: ${err.message}`);
}
const configurations = cve.configurations ?? [];
for (const node of configurations) {
for (const match of node.nodes?.flatMap(n => n.cpeMatch ?? []) ?? []) {
const cpe = match.criteria;
const vulnerable = match.vulnerable ? 1 : 0;
const start = match.versionStartIncluding || match.versionStartExcluding || null;
const end = match.versionEndIncluding || match.versionEndExcluding || null;
const { vendor, product, version } = extractCpeParts(cpe);
try {
await DB.execute(
`INSERT INTO cpe_matches (cve_id, cpe_uri, version_start, version_end, vulnerable, vendor, product, version)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
[cveId, cpe, start, end, vulnerable, vendor, product, version]
);
} catch (err) {
log(`⚠️ Error inserting CPE for CVE ${cveId}: ${err.message}`);
}
}
}
}
async function getMostRecentModifiedDateFromDB() {
const [rows] = await DB.query(`SELECT MAX(last_modified_date) AS lastMod FROM cves`);
const lastMod = rows[0]?.lastMod;
return lastMod ? new Date(lastMod).toISOString() : '2020-01-01T00:00:00.000Z';
}
async function importCVEFeed() {
const now = new Date();
const endDate = now.toISOString();
const startDateObj = new Date(now);
startDateObj.setDate(startDateObj.getDate() - MAX_RANGE_DAYS);
const startDate = startDateObj.toISOString();
log(`🚀 CVE sync started`);
log(`🔄 Initializing script...`);
log(`📍 Launching script`);
log(`📅 Starting CVE sync from ${startDate} to ${endDate}`);
const humanStart = formatShortDate(startDate);
const humanEnd = formatShortDate(endDate);
log(`📡 Fetching modified CVEs from ${humanStart} to ${humanEnd}...`);
let startIndex = 0;
let totalResults = Infinity;
let pageCount = 0;
do {
const data = await fetchCVEPage(startIndex, startDate, endDate);
const vulnerabilities = data.vulnerabilities || [];
totalResults = data.totalResults ?? vulnerabilities.length;
if (vulnerabilities.length === 0) {
log(`⚠️ No CVEs returned at index ${startIndex}`);
break;
}
log(`📄 Page ${++pageCount} — Processing ${vulnerabilities.length} CVEs from index ${startIndex} of ~${totalResults}`);
for (const vuln of vulnerabilities) {
await processCVE(vuln);
}
startIndex += RESULTS_PER_PAGE;
await new Promise((r) => setTimeout(r, 6000));
} while (startIndex < totalResults);
log('✅ CVE import complete!');
await DB.end();
logFile.end();
}
async function importCVEFeedBackfill() {
const now = new Date();
const resumeFrom = loadLastSyncedDate();
let startFrom = resumeFrom ? new Date(resumeFrom) : now;
const MAX_RANGE_DAYS = 120;
log(resumeFrom
? `🔁 Resuming CVE backfill from ${formatShortDate(startFrom.toISOString())}`
: `⏮️ Starting CVE backfill from today (${formatShortDate(startFrom.toISOString())})`
);
while (true) {
const end = new Date(startFrom);
const start = new Date(startFrom);
start.setDate(start.getDate() - MAX_RANGE_DAYS + 1); // 120-day window
const startISO = start.toISOString();
const endISO = end.toISOString();
const humanRange = `${formatShortDate(startISO)} to ${formatShortDate(endISO)}`;
log(`📡 Fetching published CVEs from ${humanRange}...`);
let startIndex = 0;
let totalResults = Infinity;
let pageCount = 0;
try {
do {
const data = await fetchCVEPage(startIndex, startISO, endISO);
const vulnerabilities = data.vulnerabilities || [];
totalResults = data.totalResults ?? vulnerabilities.length;
if (vulnerabilities.length === 0) {
log(`⚠️ No CVEs returned for ${humanRange} at index ${startIndex}`);
break;
}
log(`📄 Page ${++pageCount} — ${vulnerabilities.length} CVEs from index ${startIndex}`);
for (const vuln of vulnerabilities) {
await processCVE(vuln);
}
startIndex += RESULTS_PER_PAGE;
await new Promise((r) => setTimeout(r, 6000));
} while (startIndex < totalResults);
// Move the window backward
saveLastSyncedDate(start.toISOString());
startFrom = start;
} catch (err) {
log(`❌ Error during ${humanRange}: ${err.message}`);
break;
}
if (start < new Date('2002-01-01')) {
log(`🛑 Reached earliest supported CVE publication date — halting backfill.`);
break;
}
}
log('✅ CVE backfill complete!');
await DB.end();
logFile.end();
}
//importCVEFeed().catch((err) => {
importCVEFeedBackfill(9000) // ~25 years (goes back to 2000)
.catch((err) => {
log(`❌ Fatal error during import: ${err.message}`);
logFile.end();
});

79
scripts/fetchKEV.js Normal file
View File

@@ -0,0 +1,79 @@
// fetchKEV.js
import fs from 'fs';
import axios from 'axios';
import mysql from 'mysql2/promise';
const KEV_URL = 'https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json';
const DB = await mysql.createConnection({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
});
fs.writeFileSync('kev-sync.log', '', { flag: 'w' }); // 👈 Reset the log file cleanly
function log(msg) {
const now = new Date().toLocaleString('en-AU', {
day: '2-digit', month: 'short', year: 'numeric',
hour: '2-digit', minute: '2-digit', second: '2-digit', hour12: true
}).replace(/\b(AM|PM)\b/, m => m.toLowerCase());
const line = `[${now}] ${msg}`;
console.log(line);
fs.appendFileSync('kev-sync.log', `${line}\n`);
}
function safeDate(input) {
return input ? new Date(input).toISOString().slice(0, 10) : null;
}
async function fetchAndInsertKEVs() {
log('🚀 Starting KEV fetch from CISA...');
const { data } = await axios.get(KEV_URL);
const vulns = data.vulnerabilities;
let inserted = 0;
for (const v of vulns) {
try {
await DB.execute(`
INSERT INTO kev_catalog (
cve_id, vendor_project, product, vulnerability_name,
date_added, short_description, required_action, due_date,
known_ransomware_campaign_use, notes
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
date_added = VALUES(date_added),
short_description = VALUES(short_description),
required_action = VALUES(required_action),
due_date = VALUES(due_date),
known_ransomware_campaign_use = VALUES(known_ransomware_campaign_use),
notes = VALUES(notes)
`, [
v.cveID,
v.vendorProject,
v.product,
v.vulnerabilityName,
safeDate(v.dateAdded),
v.shortDescription,
v.requiredAction,
safeDate(v.dueDate),
v.knownRansomwareCampaignUse ?? null,
v.notes ?? null
]);
inserted++;
} catch (err) {
log(`❌ Failed to insert ${v.cveID}: ${err.message}`);
}
}
log(`✅ Finished KEV sync. Inserted/updated: ${inserted}`);
await DB.end();
}
fetchAndInsertKEVs().catch(err => {
log(`❌ Uncaught error: ${err.message}`);
DB.end();
});

193
scripts/fetchMSRC.js Normal file
View File

@@ -0,0 +1,193 @@
#!/usr/bin/env node
import axios from 'axios';
import mysql from 'mysql2/promise';
import dotenv from 'dotenv';
import readline from 'readline';
dotenv.config({ path: '.env.local' });
const DB = await mysql.createConnection({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
});
function formatDate(isoString) {
if (!isoString) return null;
const date = new Date(isoString);
return date.toISOString().slice(0, 19).replace('T', ' ');
}
function getPreviousMonthID() {
const now = new Date();
now.setMonth(now.getMonth() - 1);
const year = now.getFullYear();
const month = now.toLocaleString('en-US', { month: 'short' }); // e.g. "Mar"
return `${year}-${month}`;
}
function getLocalizedText(entry, fallback = '') {
if (!entry || typeof entry !== 'object') return fallback;
// Try to get English (US) first
if (entry['en-US']?.Value) return entry['en-US'].Value;
// Fallback: get first entry with a Value field
for (const key in entry) {
if (entry[key]?.Value) {
return entry[key].Value;
}
}
return fallback;
}
function extractDescription(notes = [], fallback = '') {
const note = notes.find(n =>
n.Title === 'Description' &&
n.Type === 2 &&
typeof n.Value === 'string' &&
n.Value.trim().length > 0
);
return note?.Value || fallback;
}
function extractAffectedProducts(vuln) {
const fixed = vuln.ProductStatuses?.Fixed ?? [];
const known = vuln.ProductStatuses?.KnownAffected ?? [];
const underInvestigation = vuln.ProductStatuses?.UnderInvestigation ?? [];
return [...new Set([...fixed, ...known, ...underInvestigation])].join(', ');
}
async function fetchCVRFDoc(cvrfId) {
const url = `https://api.msrc.microsoft.com/cvrf/v3.0/cvrf/${cvrfId}`;
try {
const res = await axios.get(url, {
headers: { 'Accept': 'application/json' }
});
return res.data;
} catch (err) {
console.error(`❌ Failed to fetch CVRF for ${cvrfId}:`, err.message);
return null;
}
}
async function storeVulnerability(v, published, modified) {
const cve = v.CVE;
console.log(`⏳ Saving ${cve}...`);
const title = getLocalizedText(v.Title, '');
const description = extractDescription(v.Notes, title);
const affectedProducts = extractAffectedProducts(v);
const severity = v.Threats?.find(t => t.Type === 'Severity')?.Description?.Value || '';
const impact = v.Threats?.find(t => t.Type === 'Impact')?.Description?.Value || '';
const exploitability = v.Threats?.find(t => t.Type === 'Exploitability')?.Description?.Value || '';
const cvssSet = v.CVSSScoreSets?.[0];
const cvssScore = cvssSet?.BaseScore || null;
const cvssVector = cvssSet?.Vector || '';
const cweList = Array.isArray(v.CWE) ? v.CWE : [];
if (!title) {
console.warn(`⚠️ No title found for ${cve}`);
console.debug('Title raw:', JSON.stringify(v.Title, null, 2));
}
if (!description) {
console.warn(`⚠️ No description for ${cve}`);
console.debug('Notes raw:', JSON.stringify(v.Notes, null, 2));
}
try {
// Insert or update the main CVE entry (no cwe column now)
await DB.execute(
`INSERT INTO microsoft_cves (cve_id, title, severity, published_date, last_modified_date,
affected_products, description, exploitability, cvss_score, cvss_vector)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE last_modified_date = VALUES(last_modified_date),
affected_products = VALUES(affected_products),
severity = VALUES(severity),
description = VALUES(description),
exploitability = VALUES(exploitability),
cvss_score = VALUES(cvss_score),
cvss_vector = VALUES(cvss_vector)`,
[cve, title, severity, published, modified, affectedProducts, description, exploitability, cvssScore, cvssVector]
);
// Insert CWE entries and links
for (const { ID: cweId, Value: cweName } of cweList) {
if (!cweId) continue;
// Insert into cwe_entries if not already present
await DB.execute(
`INSERT IGNORE INTO cwe_entries (cwe_id, cwe_name) VALUES (?, ?)`,
[cweId, cweName || null]
);
// Link CVE to CWE
await DB.execute(
`INSERT IGNORE INTO cve_cwe (cve_id, cwe_id) VALUES (?, ?)`,
[cve, cweId]
);
}
console.log(`✅ Saved ${cve} — ${severity}, ${cvssScore ?? '?'}`);
} catch (err) {
console.error(`❌ DB error for ${cve}:`, err.message);
}
}
function waitForKeypress() {
return new Promise((resolve) => {
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout
});
rl.question('🔄 Press any key to continue to the next 10 CVEs...', () => {
rl.close();
resolve();
});
});
}
async function importPreviousMonth() {
const cvrfId = getPreviousMonthID();
console.log(`📡 Fetching CVEs for ${cvrfId}...`);
const data = await fetchCVRFDoc(cvrfId);
if (!data) {
console.log("❌ No data returned from MSRC.");
return;
}
const published = formatDate(data.DocumentTracking?.InitialReleaseDate);
const modified = formatDate(data.DocumentTracking?.CurrentReleaseDate);
const vulns = data.Vulnerability || [];
console.log(`📋 Found ${vulns.length} vulnerabilities to import.`);
const batchSize = 10;
for (let i = 0; i < vulns.length; i += batchSize) {
const batch = vulns.slice(i, i + batchSize);
console.log(`🔄 Processing batch ${i / batchSize + 1} (${batch.length} items)...`);
for (const v of batch) {
await storeVulnerability(v, published, modified);
}
}
console.log(`✅ Finished importing ${vulns.length} CVEs for ${cvrfId}`);
await DB.end();
}
console.log("🚀 Starting MSRC sync now...");
await importPreviousMonth();

5
scripts/kev-sync.log Normal file
View File

@@ -0,0 +1,5 @@
[29 Apr 2025, 11:56:08 am] 🚀 Starting KEV fetch from CISA...
[29 Apr 2025, 11:56:08 am] 🚀 Starting KEV fetch from CISA...
[29 Apr 2025, 11:56:08 am] ✅ Finished KEV sync. Inserted/updated: 1326
[29 Apr 2025, 11:56:08 am] ✅ Finished KEV sync. Inserted/updated: 1326
[29 Apr 2025, 11:56:08 am] ✅ fetchKEV.js finished with exit code: 0

6978
scripts/msrc-sync.log Normal file
View File

File diff suppressed because it is too large Load Diff

13
scripts/runSpringSync_Manual.js Normal file
View File

@@ -0,0 +1,13 @@
// runSpringSync_Manual.js
import axios from 'axios';
async function triggerSpringSync() {
try {
const res = await axios.post('https://sys.psg.net.au:8443/api/admin/scripts/run-scheduled-sync');
console.log(res.data);
} catch (err) {
console.error("❌ Failed to trigger:", err.response?.data || err.message);
}
}
triggerSpringSync();