1
0
Fork 0
mirror of https://github.com/imjasonh/rustvulncheck synced 2026-07-08 04:08:07 +00:00

Add Phase 1: Offline Database Enrichment pipeline

Build the first phase of cargo deep-audit - a reachability-based vulnerability
scanner for Rust. This pipeline:

- Clones/caches the RustSec advisory-db and parses TOML advisory files
- Extracts crate names, patched versions, CVE/RUSTSEC IDs, and GitHub refs
- Fetches patch diffs via GitHub API (PRs resolve to merge commits)
- Analyzes unified diffs to extract modified Rust function signatures
- Outputs an enriched JSON database mapping advisory IDs to function signatures

Modules:
- advisory.rs: TOML parser with GitHub reference extraction
- github.rs: GitHub API client for commit/PR diffs
- diff_analyzer.rs: Unified diff parser extracting fn signatures from hunks
- db.rs: JSON output database structure
- main.rs: CLI orchestrator with clap

https://claude.ai/code/session_01AeNG3herWfKhos9uZVUY5d
This commit is contained in:
Claude 2026-03-25 01:27:41 +00:00
parent b0a0d8599d
commit 430816c717
No known key found for this signature in database
12 changed files with 2887 additions and 0 deletions

204
src/advisory.rs Normal file
View file

@ -0,0 +1,204 @@
//! Parser for RustSec advisory-db TOML files.
use anyhow::{Context, Result};
use serde::Deserialize;
use std::path::Path;
use walkdir::WalkDir;
/// Raw TOML structure of a RustSec advisory file.
#[derive(Debug, Deserialize)]
pub struct AdvisoryFile {
pub advisory: AdvisoryMeta,
#[serde(default)]
pub versions: VersionInfo,
}
#[derive(Debug, Deserialize)]
pub struct AdvisoryMeta {
pub id: String,
#[serde(default)]
pub package: String,
#[serde(default)]
pub date: String,
#[serde(default)]
pub url: Option<String>,
#[serde(default)]
pub references: Vec<String>,
#[serde(default)]
pub title: Option<String>,
#[serde(default)]
pub description: Option<String>,
}
#[derive(Debug, Default, Deserialize)]
pub struct VersionInfo {
#[serde(default)]
pub patched: Vec<String>,
#[serde(default)]
pub unaffected: Vec<String>,
}
/// Parsed advisory with extracted GitHub references.
#[derive(Debug, Clone)]
pub struct Advisory {
pub id: String,
pub package: String,
pub date: String,
pub title: String,
pub patched_versions: Vec<String>,
pub github_urls: Vec<String>,
}
impl Advisory {
/// Extract GitHub PR/issue/commit URLs from advisory references and URL fields.
pub fn github_refs(&self) -> Vec<GithubRef> {
let mut refs = Vec::new();
for url in &self.github_urls {
if let Some(r) = GithubRef::parse(url) {
refs.push(r);
}
}
refs
}
}
/// A parsed GitHub reference (PR, issue, or commit).
#[derive(Debug, Clone)]
pub enum GithubRef {
PullRequest {
owner: String,
repo: String,
number: u64,
},
Issue {
owner: String,
repo: String,
number: u64,
},
Commit {
owner: String,
repo: String,
sha: String,
},
}
impl GithubRef {
pub fn parse(url: &str) -> Option<Self> {
let url = url.trim_end_matches('/');
// Match: github.com/owner/repo/pull/123
if let Some(caps) = regex::Regex::new(
r"github\.com/([^/]+)/([^/]+)/pull/(\d+)",
)
.ok()?
.captures(url)
{
return Some(GithubRef::PullRequest {
owner: caps[1].to_string(),
repo: caps[2].to_string(),
number: caps[3].parse().ok()?,
});
}
// Match: github.com/owner/repo/issues/123
if let Some(caps) = regex::Regex::new(
r"github\.com/([^/]+)/([^/]+)/issues/(\d+)",
)
.ok()?
.captures(url)
{
return Some(GithubRef::Issue {
owner: caps[1].to_string(),
repo: caps[2].to_string(),
number: caps[3].parse().ok()?,
});
}
// Match: github.com/owner/repo/commit/<sha>
if let Some(caps) = regex::Regex::new(
r"github\.com/([^/]+)/([^/]+)/commit/([0-9a-f]+)",
)
.ok()?
.captures(url)
{
return Some(GithubRef::Commit {
owner: caps[1].to_string(),
repo: caps[2].to_string(),
sha: caps[3].to_string(),
});
}
None
}
}
/// Scan a directory of RustSec advisory TOML files and parse them.
pub fn parse_advisory_db(db_path: &Path) -> Result<Vec<Advisory>> {
let crates_dir = db_path.join("crates");
if !crates_dir.exists() {
anyhow::bail!(
"advisory-db 'crates' directory not found at {}",
crates_dir.display()
);
}
let mut advisories = Vec::new();
for entry in WalkDir::new(&crates_dir)
.into_iter()
.filter_map(|e| e.ok())
{
let path = entry.path();
if path.extension().map_or(true, |ext| ext != "toml") {
continue;
}
match parse_advisory_file(path) {
Ok(adv) => advisories.push(adv),
Err(e) => {
eprintln!("Warning: skipping {}: {}", path.display(), e);
}
}
}
// Sort by date descending (most recent first)
advisories.sort_by(|a, b| b.date.cmp(&a.date));
Ok(advisories)
}
fn parse_advisory_file(path: &Path) -> Result<Advisory> {
let content =
std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display()))?;
let file: AdvisoryFile =
toml::from_str(&content).with_context(|| format!("parsing {}", path.display()))?;
let mut github_urls = Vec::new();
// Collect URLs from the `url` field
if let Some(ref url) = file.advisory.url {
if url.contains("github.com") {
github_urls.push(url.clone());
}
}
// Collect URLs from `references`
for r in &file.advisory.references {
if r.contains("github.com") {
github_urls.push(r.clone());
}
}
Ok(Advisory {
id: file.advisory.id,
package: file.advisory.package,
date: file.advisory.date,
title: file
.advisory
.title
.unwrap_or_else(|| "(no title)".to_string()),
patched_versions: file.versions.patched,
github_urls,
})
}

48
src/db.rs Normal file
View file

@ -0,0 +1,48 @@
//! Output database: maps advisory IDs to vulnerable function signatures.
use serde::Serialize;
use std::path::Path;
/// The enriched vulnerability database.
#[derive(Debug, Serialize)]
pub struct VulnDb {
pub generated_at: String,
pub entries: Vec<VulnEntry>,
}
/// A single vulnerability entry with function-level detail.
#[derive(Debug, Serialize)]
pub struct VulnEntry {
pub advisory_id: String,
pub package: String,
pub title: String,
pub date: String,
pub patched_versions: Vec<String>,
pub commit_sha: Option<String>,
pub vulnerable_symbols: Vec<crate::diff_analyzer::VulnerableSymbol>,
}
impl VulnDb {
pub fn new() -> Self {
let now = chrono_lite_now();
Self {
generated_at: now,
entries: Vec::new(),
}
}
pub fn write_json(&self, path: &Path) -> anyhow::Result<()> {
let json = serde_json::to_string_pretty(self)?;
std::fs::write(path, json)?;
Ok(())
}
}
/// Simple timestamp without pulling in chrono.
fn chrono_lite_now() -> String {
use std::time::SystemTime;
let duration = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default();
format!("unix:{}", duration.as_secs())
}

201
src/diff_analyzer.rs Normal file
View file

@ -0,0 +1,201 @@
//! Analyze unified diffs to extract modified Rust function signatures.
use regex::Regex;
use crate::github::PatchDiff;
/// A vulnerable symbol extracted from a patch diff.
#[derive(Debug, Clone, serde::Serialize)]
pub struct VulnerableSymbol {
/// The file path in the repository
pub file: String,
/// The inferred fully-qualified function name (best effort)
pub function: String,
/// Whether this is a new function, modified function, or deleted function
pub change_type: ChangeType,
}
#[derive(Debug, Clone, serde::Serialize)]
pub enum ChangeType {
Modified,
Added,
Deleted,
}
/// Extract function signatures from a patch diff.
///
/// Strategy:
/// 1. Look at unified diff hunk headers (`@@ ... @@ fn ...`) which often contain
/// the enclosing function name.
/// 2. Look at added/removed lines containing `fn ` declarations.
/// 3. Infer module path from the file path.
pub fn extract_symbols(diff: &PatchDiff) -> Vec<VulnerableSymbol> {
let mut symbols = Vec::new();
let fn_decl_re = Regex::new(
r"(?:pub\s+(?:\(crate\)\s+)?)?(?:unsafe\s+)?(?:async\s+)?fn\s+([a-zA-Z_][a-zA-Z0-9_]*)",
)
.unwrap();
let hunk_header_re = Regex::new(r"^@@.*@@\s*(.*)$").unwrap();
let impl_re =
Regex::new(r"impl(?:<[^>]*>)?\s+(?:([a-zA-Z_][a-zA-Z0-9_:]*)\s+for\s+)?([a-zA-Z_][a-zA-Z0-9_:<>, ]*)").unwrap();
for file_patch in &diff.files {
let module_path = file_path_to_module(&file_patch.filename);
let mut current_context_fn: Option<String> = None;
let mut current_impl_type: Option<String> = None;
for line in file_patch.patch.lines() {
// Track hunk headers - they often show the enclosing function
if let Some(caps) = hunk_header_re.captures(line) {
let context = &caps[1];
// Check if the hunk header contains an impl block
if let Some(icaps) = impl_re.captures(context) {
current_impl_type = Some(icaps[2].trim().to_string());
current_context_fn = None;
}
// Check if hunk header mentions a fn
if let Some(fcaps) = fn_decl_re.captures(context) {
current_context_fn = Some(fcaps[1].to_string());
}
continue;
}
// Track impl blocks in context lines
if line.starts_with(' ') || line.starts_with('+') || line.starts_with('-') {
let content = &line[1..];
if let Some(icaps) = impl_re.captures(content) {
if !content.trim_start().starts_with("//") {
current_impl_type = Some(icaps[2].trim().to_string());
current_context_fn = None;
}
}
}
// Look for fn declarations in changed lines
let is_added = line.starts_with('+') && !line.starts_with("+++");
let is_removed = line.starts_with('-') && !line.starts_with("---");
if (is_added || is_removed) && line.contains("fn ") {
let content = &line[1..];
// Skip comments
if content.trim_start().starts_with("//") || content.trim_start().starts_with("*")
{
continue;
}
if let Some(caps) = fn_decl_re.captures(content) {
let fn_name = &caps[1];
let qualified = qualify_fn_name(&module_path, &current_impl_type, fn_name);
let change_type = if is_added {
ChangeType::Added
} else {
ChangeType::Deleted
};
symbols.push(VulnerableSymbol {
file: file_patch.filename.clone(),
function: qualified,
change_type,
});
}
} else if is_added || is_removed {
// Changed line inside a known function context
let content = &line[1..];
if content.trim().is_empty() || content.trim_start().starts_with("//") {
continue;
}
if let Some(ref ctx_fn) = current_context_fn {
let qualified =
qualify_fn_name(&module_path, &current_impl_type, ctx_fn);
if !symbols.iter().any(|s| s.function == qualified) {
symbols.push(VulnerableSymbol {
file: file_patch.filename.clone(),
function: qualified,
change_type: ChangeType::Modified,
});
}
}
}
// Update current function context from context/added lines
if !line.starts_with('-') {
let content = if line.starts_with('+') || line.starts_with(' ') {
&line[1..]
} else {
line
};
if content.contains("fn ") && !content.trim_start().starts_with("//") {
if let Some(caps) = fn_decl_re.captures(content) {
current_context_fn = Some(caps[1].to_string());
}
}
}
}
}
symbols
}
/// Convert a file path like `src/http/request.rs` to a module path like `http::request`.
fn file_path_to_module(path: &str) -> String {
let path = path
.trim_start_matches("src/")
.trim_end_matches(".rs")
.trim_end_matches("/mod")
.trim_end_matches("/lib");
path.replace('/', "::")
}
/// Build a qualified function name from module path, optional impl type, and fn name.
fn qualify_fn_name(module: &str, impl_type: &Option<String>, fn_name: &str) -> String {
match impl_type {
Some(ty) => {
// Clean up generic parameters for the type name
let clean_ty = ty
.split('<')
.next()
.unwrap_or(ty)
.trim()
.to_string();
format!("{}::{}::{}", module, clean_ty, fn_name)
}
None => format!("{}::{}", module, fn_name),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_file_path_to_module() {
assert_eq!(file_path_to_module("src/http/request.rs"), "http::request");
assert_eq!(file_path_to_module("src/lib.rs"), "lib");
assert_eq!(
file_path_to_module("src/net/tcp/mod.rs"),
"net::tcp"
);
}
#[test]
fn test_extract_fn_from_diff() {
let diff = PatchDiff {
commit_sha: "abc123".to_string(),
files: vec![crate::github::FilePatch {
filename: "src/http/request.rs".to_string(),
patch: r#"@@ -10,6 +10,8 @@ impl Request {
pub fn parse(buf: &[u8]) -> Result<Self> {
- let old_code = true;
+ let new_code = true;
+ let extra = false;
}
"#
.to_string(),
}],
};
let symbols = extract_symbols(&diff);
assert!(!symbols.is_empty());
assert!(symbols.iter().any(|s| s.function.contains("parse")));
}
}

142
src/github.rs Normal file
View file

@ -0,0 +1,142 @@
//! GitHub API client for fetching patch diffs.
use anyhow::{Context, Result};
use reqwest::blocking::Client;
use reqwest::header::{ACCEPT, AUTHORIZATION, USER_AGENT};
use serde::Deserialize;
use crate::advisory::GithubRef;
pub struct GithubClient {
client: Client,
token: Option<String>,
}
#[derive(Debug, Deserialize)]
struct PrResponse {
merge_commit_sha: Option<String>,
}
#[derive(Debug, Deserialize)]
struct CommitFile {
filename: String,
patch: Option<String>,
}
/// A fetched diff from GitHub, containing per-file patches.
#[derive(Debug, Clone)]
pub struct PatchDiff {
pub commit_sha: String,
pub files: Vec<FilePatch>,
}
#[derive(Debug, Clone)]
pub struct FilePatch {
pub filename: String,
pub patch: String,
}
impl GithubClient {
pub fn new(token: Option<String>) -> Self {
Self {
client: Client::new(),
token,
}
}
fn get(&self, url: &str) -> reqwest::blocking::RequestBuilder {
let mut req = self
.client
.get(url)
.header(USER_AGENT, "cargo-deep-audit/0.1")
.header(ACCEPT, "application/vnd.github.v3+json");
if let Some(ref token) = self.token {
req = req.header(AUTHORIZATION, format!("Bearer {}", token));
}
req
}
/// Fetch the diff for a GitHub reference (PR, commit, or issue).
/// For PRs, resolves the merge commit first. For issues, returns None.
pub fn fetch_diff(&self, gh_ref: &GithubRef) -> Result<Option<PatchDiff>> {
match gh_ref {
GithubRef::Commit { owner, repo, sha } => {
let diff = self.fetch_commit_diff(owner, repo, sha)?;
Ok(Some(diff))
}
GithubRef::PullRequest {
owner,
repo,
number,
} => {
// Get the merge commit SHA from the PR
let url = format!(
"https://api.github.com/repos/{}/{}/pulls/{}",
owner, repo, number
);
let resp: PrResponse = self
.get(&url)
.send()
.context("fetching PR metadata")?
.error_for_status()
.context("PR API error")?
.json()
.context("parsing PR response")?;
if let Some(sha) = resp.merge_commit_sha {
let diff = self.fetch_commit_diff(owner, repo, &sha)?;
Ok(Some(diff))
} else {
eprintln!(" PR {}/{}/pull/{} has no merge commit", owner, repo, number);
Ok(None)
}
}
GithubRef::Issue { .. } => {
// Issues don't have diffs directly; skip.
Ok(None)
}
}
}
fn fetch_commit_diff(&self, owner: &str, repo: &str, sha: &str) -> Result<PatchDiff> {
let url = format!(
"https://api.github.com/repos/{}/{}/commits/{}",
owner, repo, sha
);
let resp = self
.get(&url)
.send()
.context("fetching commit")?
.error_for_status()
.context("commit API error")?;
#[derive(Deserialize)]
struct CommitResponse {
sha: String,
files: Option<Vec<CommitFile>>,
}
let commit: CommitResponse = resp.json().context("parsing commit response")?;
let files = commit
.files
.unwrap_or_default()
.into_iter()
.filter(|f| f.filename.ends_with(".rs"))
.filter_map(|f| {
f.patch.map(|patch| FilePatch {
filename: f.filename,
patch,
})
})
.collect();
Ok(PatchDiff {
commit_sha: commit.sha,
files,
})
}
}

197
src/main.rs Normal file
View file

@ -0,0 +1,197 @@
mod advisory;
mod db;
mod diff_analyzer;
mod github;
use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use clap::Parser;
use advisory::{parse_advisory_db, Advisory};
use db::{VulnDb, VulnEntry};
use diff_analyzer::extract_symbols;
use github::GithubClient;
/// cargo deep-audit: Phase 1 - Offline Database Enrichment
///
/// Clones the RustSec advisory-db, parses advisories, fetches patch diffs
/// from GitHub, and extracts vulnerable function signatures.
#[derive(Parser, Debug)]
#[command(name = "cargo-deep-audit", version, about)]
struct Args {
/// Path to a local clone of the RustSec advisory-db.
/// If not provided, the repo will be cloned to a temp directory.
#[arg(long)]
advisory_db: Option<PathBuf>,
/// Maximum number of advisories to process (most recent first).
#[arg(long, default_value = "10")]
limit: usize,
/// Output JSON file path.
#[arg(long, default_value = "vuln_db.json")]
output: PathBuf,
/// GitHub API token (can also be set via GITHUB_TOKEN env var).
#[arg(long, env = "GITHUB_TOKEN")]
github_token: Option<String>,
/// Include all advisories, even those without GitHub references.
#[arg(long)]
include_all: bool,
}
fn main() -> Result<()> {
let args = Args::parse();
// Step 1: Get the advisory-db
let db_path = match &args.advisory_db {
Some(path) => {
println!("Using local advisory-db at {}", path.display());
path.clone()
}
None => {
let tmp = PathBuf::from("/tmp/advisory-db");
clone_advisory_db(&tmp)?;
tmp
}
};
// Step 2: Parse advisories
println!("Parsing advisory database...");
let all_advisories = parse_advisory_db(&db_path)?;
println!("Found {} total advisories", all_advisories.len());
// Filter to advisories with GitHub refs unless --include-all
let candidates: Vec<&Advisory> = if args.include_all {
all_advisories.iter().collect()
} else {
all_advisories
.iter()
.filter(|a| !a.github_urls.is_empty())
.collect()
};
println!(
"{} advisories have GitHub references",
candidates.len()
);
let to_process = &candidates[..candidates.len().min(args.limit)];
println!("Processing {} advisories...\n", to_process.len());
// Step 3: Fetch diffs and extract symbols
let gh = GithubClient::new(args.github_token);
let mut vuln_db = VulnDb::new();
for (i, adv) in to_process.iter().enumerate() {
println!(
"[{}/{}] {} ({}) - {}",
i + 1,
to_process.len(),
adv.id,
adv.package,
adv.title
);
let gh_refs = adv.github_refs();
if gh_refs.is_empty() {
println!(" No parseable GitHub refs, skipping");
continue;
}
let mut entry = VulnEntry {
advisory_id: adv.id.clone(),
package: adv.package.clone(),
title: adv.title.clone(),
date: adv.date.clone(),
patched_versions: adv.patched_versions.clone(),
commit_sha: None,
vulnerable_symbols: Vec::new(),
};
// Try each ref until we get a diff
for gh_ref in &gh_refs {
match gh.fetch_diff(gh_ref) {
Ok(Some(diff)) => {
println!(
" Fetched commit {} ({} .rs files)",
&diff.commit_sha[..7.min(diff.commit_sha.len())],
diff.files.len()
);
entry.commit_sha = Some(diff.commit_sha.clone());
let symbols = extract_symbols(&diff);
if symbols.is_empty() {
println!(" No function signatures extracted from diff");
} else {
println!(" Extracted {} symbols:", symbols.len());
for sym in &symbols {
println!(" - {} ({:?})", sym.function, sym.change_type);
}
}
entry.vulnerable_symbols = symbols;
break;
}
Ok(None) => {
println!(" Ref returned no diff, trying next...");
}
Err(e) => {
eprintln!(" Error fetching diff: {}", e);
}
}
}
vuln_db.entries.push(entry);
println!();
}
// Step 4: Write output
let entries_with_symbols = vuln_db
.entries
.iter()
.filter(|e| !e.vulnerable_symbols.is_empty())
.count();
println!(
"Done! {} of {} entries have extracted symbols.",
entries_with_symbols,
vuln_db.entries.len()
);
vuln_db.write_json(&args.output)?;
println!("Wrote enriched database to {}", args.output.display());
Ok(())
}
fn clone_advisory_db(dest: &Path) -> Result<()> {
if dest.exists() {
println!(
"Advisory-db already cached at {}, pulling updates...",
dest.display()
);
let repo = git2::Repository::open(dest).context("opening cached advisory-db")?;
// Fetch origin/main
let mut remote = repo.find_remote("origin")?;
remote.fetch(&["main"], None, None)?;
// Fast-forward to origin/main
let fetch_head = repo.find_reference("refs/remotes/origin/main")?;
let commit = repo.reference_to_annotated_commit(&fetch_head)?;
let (analysis, _) = repo.merge_analysis(&[&commit])?;
if analysis.is_fast_forward() {
let mut reference = repo.find_reference("refs/heads/main")?;
reference.set_target(commit.id(), "fast-forward")?;
repo.set_head("refs/heads/main")?;
repo.checkout_head(Some(git2::build::CheckoutBuilder::new().force()))?;
println!("Fast-forwarded to latest.");
} else {
println!("Already up to date.");
}
return Ok(());
}
println!("Cloning RustSec advisory-db to {}...", dest.display());
git2::Repository::clone("https://github.com/rustsec/advisory-db.git", dest)
.context("cloning advisory-db")?;
println!("Clone complete.");
Ok(())
}