mirror of
https://github.com/imjasonh/rustvulncheck
synced 2026-07-08 04:08:07 +00:00
Add --re-enrich flag to reprocess existing DB entries with current code
The old --existing-db flag skipped already-enriched entries by advisory ID, so improvements to diff_analyzer.rs had no effect on the committed vuln_db.json. The new --re-enrich flag re-fetches diffs for existing entries and re-extracts symbols using the current code, updating them in-place. The CI enrich job now passes --re-enrich so that code changes to the symbol extraction pipeline are reflected in the PR's vuln_db.json diff. https://claude.ai/code/session_01P1LKP6aqGt68rQAXrF6kSE
This commit is contained in:
Claude
parent
edfd55f0dd
commit
0b933a662d
3 changed files with 165 additions and 39 deletions
189
src/main.rs
189
src/main.rs
|
|
@ -65,6 +65,12 @@ struct EnrichArgs {
|
|||
#[arg(long)]
|
||||
existing_db: Option<PathBuf>,
|
||||
|
||||
/// Re-enrich existing entries by re-fetching diffs and re-extracting symbols
|
||||
/// with the current code. Requires --existing-db. Processes up to --limit entries
|
||||
/// that have a commit_sha.
|
||||
#[arg(long)]
|
||||
re_enrich: bool,
|
||||
|
||||
/// Stop processing after this many seconds (for CI time-boxing).
|
||||
#[arg(long)]
|
||||
timeout_secs: Option<u64>,
|
||||
|
|
@ -153,17 +159,133 @@ fn run_enrich(args: EnrichArgs) -> Result<()> {
|
|||
_ => VulnDb::new(),
|
||||
};
|
||||
|
||||
// Step 3: Parse advisories
|
||||
println!("Parsing advisory database...");
|
||||
let all_advisories = parse_advisory_db(&db_path)?;
|
||||
println!("Found {} total advisories", all_advisories.len());
|
||||
|
||||
// Build advisory lookup by ID for re-enrichment
|
||||
let advisory_by_id: std::collections::HashMap<&str, &Advisory> = all_advisories
|
||||
.iter()
|
||||
.map(|a| (a.id.as_str(), a))
|
||||
.collect();
|
||||
|
||||
let gh = GithubClient::new(args.github_token);
|
||||
|
||||
// Helper closure to check timeout
|
||||
let timed_out = |start: &Instant, t: &Option<Duration>| -> bool {
|
||||
if let Some(t) = t {
|
||||
if start.elapsed() >= *t {
|
||||
println!(
|
||||
"Timeout reached after {} seconds, stopping.",
|
||||
start.elapsed().as_secs()
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
};
|
||||
|
||||
// Step 4: Re-enrich existing entries if --re-enrich is set
|
||||
let mut re_enriched_count = 0usize;
|
||||
|
||||
if args.re_enrich {
|
||||
// Find existing entries that have commit SHAs and matching advisories
|
||||
let re_enrich_indices: Vec<usize> = vuln_db
|
||||
.entries
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, e)| {
|
||||
e.commit_sha.is_some() && advisory_by_id.contains_key(e.advisory_id.as_str())
|
||||
})
|
||||
.map(|(i, _)| i)
|
||||
.collect();
|
||||
|
||||
let re_enrich_target = args.limit.min(re_enrich_indices.len());
|
||||
println!(
|
||||
"Re-enriching up to {} of {} existing entries...\n",
|
||||
re_enrich_target,
|
||||
re_enrich_indices.len()
|
||||
);
|
||||
|
||||
for &idx in &re_enrich_indices {
|
||||
if re_enriched_count >= args.limit {
|
||||
println!(
|
||||
"Reached --limit of {} re-enriched entries, stopping.",
|
||||
args.limit
|
||||
);
|
||||
break;
|
||||
}
|
||||
if timed_out(&start_time, &timeout) {
|
||||
break;
|
||||
}
|
||||
|
||||
let entry = &vuln_db.entries[idx];
|
||||
let adv = advisory_by_id[entry.advisory_id.as_str()];
|
||||
let old_symbol_count = entry.vulnerable_symbols.len();
|
||||
|
||||
println!(
|
||||
"[{}/{}] Re-enriching {} ({})",
|
||||
re_enriched_count + 1,
|
||||
re_enrich_target,
|
||||
entry.advisory_id,
|
||||
entry.package,
|
||||
);
|
||||
|
||||
let gh_refs = adv.github_refs();
|
||||
let mut new_symbols = Vec::new();
|
||||
let mut new_sha = entry.commit_sha.clone();
|
||||
|
||||
for gh_ref in &gh_refs {
|
||||
match gh.fetch_diff(gh_ref) {
|
||||
Ok(Some(diff)) => {
|
||||
println!(
|
||||
" Fetched commit {} ({} .rs files)",
|
||||
&diff.commit_sha[..7.min(diff.commit_sha.len())],
|
||||
diff.files.len()
|
||||
);
|
||||
new_sha = Some(diff.commit_sha.clone());
|
||||
new_symbols = extract_symbols(&diff);
|
||||
break;
|
||||
}
|
||||
Ok(None) => {
|
||||
println!(" Ref returned no diff, trying next...");
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(" Error fetching diff: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let new_symbol_count = new_symbols.len();
|
||||
if new_symbol_count != old_symbol_count {
|
||||
println!(
|
||||
" Symbols: {} -> {} (changed)",
|
||||
old_symbol_count, new_symbol_count
|
||||
);
|
||||
} else {
|
||||
println!(" Symbols: {} (unchanged)", new_symbol_count);
|
||||
}
|
||||
for sym in &new_symbols {
|
||||
println!(" - {} ({:?})", sym.function, sym.change_type);
|
||||
}
|
||||
|
||||
// Update the entry in-place
|
||||
let entry_mut = &mut vuln_db.entries[idx];
|
||||
entry_mut.commit_sha = new_sha;
|
||||
entry_mut.vulnerable_symbols = new_symbols;
|
||||
re_enriched_count += 1;
|
||||
println!();
|
||||
}
|
||||
}
|
||||
|
||||
// Step 5: Process new (unenriched) advisories
|
||||
let already_enriched: HashSet<String> = vuln_db
|
||||
.entries
|
||||
.iter()
|
||||
.map(|e| e.advisory_id.clone())
|
||||
.collect();
|
||||
|
||||
// Step 3: Parse advisories
|
||||
println!("Parsing advisory database...");
|
||||
let all_advisories = parse_advisory_db(&db_path)?;
|
||||
println!("Found {} total advisories", all_advisories.len());
|
||||
|
||||
// Filter to advisories with GitHub refs unless --include-all
|
||||
let candidates: Vec<&Advisory> = if args.include_all {
|
||||
all_advisories.iter().collect()
|
||||
|
|
@ -188,30 +310,28 @@ fn run_enrich(args: EnrichArgs) -> Result<()> {
|
|||
unenriched.len()
|
||||
);
|
||||
|
||||
let target = args.limit.min(unenriched.len());
|
||||
println!(
|
||||
"Processing advisories (collecting up to {} new enriched entries)...\n",
|
||||
target
|
||||
);
|
||||
let new_limit = if args.re_enrich {
|
||||
// When re-enriching, don't also add new entries (separate concerns)
|
||||
0
|
||||
} else {
|
||||
args.limit
|
||||
};
|
||||
let target = new_limit.min(unenriched.len());
|
||||
if target > 0 {
|
||||
println!(
|
||||
"Processing advisories (collecting up to {} new enriched entries)...\n",
|
||||
target
|
||||
);
|
||||
}
|
||||
|
||||
// Step 4: Fetch diffs and extract symbols
|
||||
let gh = GithubClient::new(args.github_token);
|
||||
let mut new_count = 0usize;
|
||||
|
||||
for adv in &unenriched {
|
||||
if new_count >= args.limit {
|
||||
println!("Reached --limit of {} new entries, stopping.", args.limit);
|
||||
if new_count >= new_limit {
|
||||
break;
|
||||
}
|
||||
|
||||
if let Some(t) = timeout {
|
||||
if start_time.elapsed() >= t {
|
||||
println!(
|
||||
"Timeout reached after {} seconds, stopping.",
|
||||
start_time.elapsed().as_secs()
|
||||
);
|
||||
break;
|
||||
}
|
||||
if timed_out(&start_time, &timeout) {
|
||||
break;
|
||||
}
|
||||
|
||||
let gh_refs = adv.github_refs();
|
||||
|
|
@ -277,18 +397,27 @@ fn run_enrich(args: EnrichArgs) -> Result<()> {
|
|||
// Update timestamp
|
||||
vuln_db.update_timestamp();
|
||||
|
||||
// Step 5: Write output
|
||||
// Step 6: Write output
|
||||
let entries_with_symbols = vuln_db
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|e| !e.vulnerable_symbols.is_empty())
|
||||
.count();
|
||||
println!(
|
||||
"Done! Added {} new entries ({} total, {} with symbols).",
|
||||
new_count,
|
||||
vuln_db.entries.len(),
|
||||
entries_with_symbols,
|
||||
);
|
||||
if args.re_enrich {
|
||||
println!(
|
||||
"Done! Re-enriched {} entries ({} total, {} with symbols).",
|
||||
re_enriched_count,
|
||||
vuln_db.entries.len(),
|
||||
entries_with_symbols,
|
||||
);
|
||||
} else {
|
||||
println!(
|
||||
"Done! Added {} new entries ({} total, {} with symbols).",
|
||||
new_count,
|
||||
vuln_db.entries.len(),
|
||||
entries_with_symbols,
|
||||
);
|
||||
}
|
||||
|
||||
vuln_db.write_json(&args.output)?;
|
||||
println!("Wrote enriched database to {}", args.output.display());
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue