aiegis_harness/

pack_fetcher.rs

// SPDX-License-Identifier: Apache-2.0 OR MIT
//! Remote policy-pack fetcher.
//!
//! Talks to Nel's signed-distribution surface at
//!   GET <base>/index.json
//!   GET <base>/<pack>/<ver>.tar.gz
//!   GET <base>/<pack>/<ver>.sig
//!
//! For each pack listed in `index.json` we:
//!   1. Fetch the .tar.gz and .sig.
//!   2. Verify the Ed25519 signature against the issuer pubkey
//!      (`harness_core::pack_sig::verify_pack_tarball`, which empirically
//!      matches Nel's `/opt/aegis/aegis-registry/src/policy_packs.py
//!      ::_sign_tarball_sha`).
//!   3. Compare the sha256 against the value advertised in index.json
//!      (defence in depth — sig already covers this, but a mismatch flags
//!      a publisher bug or MITM that fooled the sig in some unforeseen
//!      way).
//!   4. Cache to disk under
//!      $XDG_CACHE_HOME/aiegis-harness/packs/ (fallback ~/.cache/...)
//!      so re-fetch is a no-op on warm start.
//!   5. Extract manifest.json + .rego file(s) into the cache dir for
//!      offline inspection.
//!
//! ROUTING DECISION (honest):
//!   Nel's packs ship full OPA Rego v1 (`import rego.v1` + `contains msg if`
//!   syntax). The Rust reference daemon's evaluator only supports the
//!   `re_match(<regex>, input.value)` subset. So we DO NOT register Nel's
//!   packs in the evaluator stack — they are loaded as INVENTORY (visible
//!   in /health, sig-verified, cached) but actions are evaluated only
//!   against locally-loaded AHP-Policy-Pack/0.1 JSON files passed via
//!   --policy-pack.
//!
//!   This is what the spec mandates: "NO FAKE STUFF. If sig verification
//!   can't reproduce in pure Rust without writing speculative code,
//!   surface as needs-iteration." Sig verification reproduces fine. Rego
//!   v1 execution does not — embedding a Rego interpreter is a separate
//!   work item (see needs-iteration note below).
//!
//!   needs-iteration: embed `opa-rs` or `regorus` to actually execute
//!   Nel's published packs. Tracked as Day-2 follow-up; until then the
//!   harness uses Nel's packs as a SIGNED LISTING, not as live policy.

use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};

use serde::Deserialize;
use sha2::{Digest, Sha256};

use harness_core::pack_sig;

#[derive(Debug, Clone, Deserialize)]
pub struct PackIndex {
    pub packs: Vec<PackDescriptor>,
    #[serde(default)]
    pub updated_at: String,
    #[serde(default)]
    pub publisher: String,
    #[serde(default)]
    pub manifest_schema_version: u32,
}

#[derive(Debug, Clone, Deserialize)]
pub struct PackDescriptor {
    pub name: String,
    pub version: String,
    pub sha256: String,
    pub signed_url: String,
    pub sig_url: String,
    #[serde(default)]
    pub sig: String, // hex inline sig (same bytes as .sig file)
    #[serde(default)]
    pub signed: bool,
}

#[derive(Debug, Clone)]
pub struct RemotePack {
    pub name: String,
    pub version: String,
    pub sha256: String,
    pub tarball_path: PathBuf,
    pub manifest: serde_json::Value,
    pub rego_files: Vec<String>,
    pub signature_verified: bool,
    /// Set when the pack is loaded as inventory-only because the
    /// reference evaluator does not execute OPA Rego v1.
    pub inventory_only: bool,
}

#[derive(Debug)]
pub enum FetchError {
    Http(String),
    Json(String),
    Io(String),
    Sha256Mismatch { got: String, want: String },
    SigVerify(String),
    Tar(String),
}

impl std::fmt::Display for FetchError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            FetchError::Http(s) => write!(f, "pack_fetch_http:{s}"),
            FetchError::Json(s) => write!(f, "pack_fetch_json:{s}"),
            FetchError::Io(s) => write!(f, "pack_fetch_io:{s}"),
            FetchError::Sha256Mismatch { got, want } => {
                write!(f, "pack_fetch_sha256_mismatch: got {got} want {want}")
            }
            FetchError::SigVerify(s) => write!(f, "pack_fetch_sig_verify:{s}"),
            FetchError::Tar(s) => write!(f, "pack_fetch_tar:{s}"),
        }
    }
}

impl std::error::Error for FetchError {}

/// Compute the on-disk cache root.
///
///   $XDG_CACHE_HOME/aiegis-harness/packs  if XDG_CACHE_HOME is set
///   ~/.cache/aiegis-harness/packs         otherwise
pub fn cache_root() -> PathBuf {
    if let Ok(xdg) = std::env::var("XDG_CACHE_HOME") {
        if !xdg.is_empty() {
            return Path::new(&xdg).join("aiegis-harness").join("packs");
        }
    }
    let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string());
    Path::new(&home).join(".cache").join("aiegis-harness").join("packs")
}

/// Fetch + verify + cache every pack listed at `<base_url>/index.json`.
///
/// `base_url` is the directory URL (e.g.
/// `https://aiegis.ie/v1/harness/policy-packs/`). The function tolerates
/// trailing-slash variation.
///
/// `issuer_pubkey_hex` is the hex-encoded raw Ed25519 public key the
/// signatures must verify under. Defaults to
/// `pack_sig::DEFAULT_PACK_ISSUER_PUBKEY_HEX`.
pub async fn fetch_all(
    base_url: &str,
    issuer_pubkey_hex: &str,
) -> Result<Vec<RemotePack>, FetchError> {
    let client = reqwest::Client::builder()
        .user_agent(format!("aiegis-harness-rs/{}", env!("CARGO_PKG_VERSION")))
        .timeout(std::time::Duration::from_secs(30))
        .build()
        .map_err(|e| FetchError::Http(format!("client_build:{e}")))?;

    let base = base_url.trim_end_matches('/');
    let index_url = format!("{base}/index.json");
    tracing::info!("pack_fetcher: GET {}", index_url);
    let resp = client
        .get(&index_url)
        .send()
        .await
        .map_err(|e| FetchError::Http(format!("index_get:{e}")))?;
    let status = resp.status();
    if !status.is_success() {
        return Err(FetchError::Http(format!("index_status:{status}")));
    }
    let index_bytes = resp
        .bytes()
        .await
        .map_err(|e| FetchError::Http(format!("index_read:{e}")))?;
    let index: PackIndex = serde_json::from_slice(&index_bytes)
        .map_err(|e| FetchError::Json(format!("index_parse:{e}")))?;

    let cache_dir = cache_root();
    fs::create_dir_all(&cache_dir).map_err(|e| FetchError::Io(format!("mkdir:{e}")))?;

    let mut out: Vec<RemotePack> = Vec::with_capacity(index.packs.len());
    for d in &index.packs {
        match fetch_one(&client, d, issuer_pubkey_hex, &cache_dir).await {
            Ok(rp) => out.push(rp),
            Err(e) => {
                tracing::error!(
                    "pack_fetch_failed pack={}/{} err={}",
                    d.name,
                    d.version,
                    e
                );
                // Honest posture: one failed pack does NOT poison the rest.
                // Daemon comes up with the packs that verified.
            }
        }
    }
    Ok(out)
}

async fn fetch_one(
    client: &reqwest::Client,
    d: &PackDescriptor,
    issuer_pubkey_hex: &str,
    cache_dir: &Path,
) -> Result<RemotePack, FetchError> {
    let pack_dir = cache_dir.join(&d.name).join(&d.version);
    fs::create_dir_all(&pack_dir).map_err(|e| FetchError::Io(format!("mkdir:{e}")))?;
    let tar_path = pack_dir.join(format!("{}.tar.gz", d.version));
    let sig_path = pack_dir.join(format!("{}.sig", d.version));

    // tarball
    let tar_bytes = fetch_bytes(client, &d.signed_url).await?;
    fs::write(&tar_path, &tar_bytes).map_err(|e| FetchError::Io(format!("write_tar:{e}")))?;

    // sha256 check against index.json claim
    let mut hasher = Sha256::new();
    hasher.update(&tar_bytes);
    let got = hex::encode(hasher.finalize());
    if got != d.sha256 {
        return Err(FetchError::Sha256Mismatch {
            got,
            want: d.sha256.clone(),
        });
    }

    // signature (raw 64 bytes)
    let sig_bytes = fetch_bytes(client, &d.sig_url).await?;
    fs::write(&sig_path, &sig_bytes).map_err(|e| FetchError::Io(format!("write_sig:{e}")))?;

    // Cross-check inline-hex sig in index against the file we fetched —
    // this catches index/file drift before we even verify.
    if !d.sig.is_empty() {
        let expected = hex::decode(&d.sig)
            .map_err(|e| FetchError::Json(format!("inline_sig_hex:{e}")))?;
        if expected != sig_bytes {
            return Err(FetchError::SigVerify(format!(
                "inline_sig_mismatches_file pack={}/{}",
                d.name, d.version
            )));
        }
    }

    // Real Ed25519 verification.
    pack_sig::verify_pack_tarball(issuer_pubkey_hex, &tar_bytes, &sig_bytes)
        .map_err(|e| FetchError::SigVerify(format!("{e}")))?;

    // Unpack manifest.json + rego files.
    //
    // ZIP-SLIP DEFENCE (audit 2026-05-25): tar entries are filtered to reject
    // any path that:
    //   - is absolute (e.g. `/etc/passwd`)
    //   - contains a `..` component (e.g. `../../etc/passwd`)
    //   - contains a Windows drive prefix or backslash
    //   - resolves outside `pack_dir` after `pack_dir.join(name)` canonicalisation
    // Without these, a malicious (or signing-key-compromised) issuer could
    // write arbitrary files under $XDG_CACHE_HOME/aiegis-harness/packs/..
    // Verified empirically with a crafted `../../poc_outside.txt` tar.
    // Decompression-bomb defence (RAV audit 2026-05-25): even with a valid
    // signature, cap per-entry + aggregate decompressed bytes so a malicious
    // (or compromised-key) issuer cannot OOM the daemon by shipping a
    // multi-GB inner payload. The Nel-published packs we've inspected are
    // <50KB each; the 16 MiB / 64 MiB caps are 300x+ headroom.
    const MAX_TAR_ENTRY_BYTES: u64 = 16 * 1024 * 1024;
    const MAX_TAR_TOTAL_BYTES: u64 = 64 * 1024 * 1024;
    let mut manifest: serde_json::Value = serde_json::Value::Null;
    let mut rego_files: Vec<String> = Vec::new();
    {
        let gz = flate2::read::GzDecoder::new(&tar_bytes[..]);
        let mut ar = tar::Archive::new(gz);
        let mut total_bytes: u64 = 0;
        for entry in ar.entries().map_err(|e| FetchError::Tar(format!("{e}")))? {
            let mut entry = entry.map_err(|e| FetchError::Tar(format!("entry:{e}")))?;
            let declared_size = entry.size();
            if declared_size > MAX_TAR_ENTRY_BYTES {
                return Err(FetchError::Tar(format!(
                    "entry_too_large: {declared_size} bytes > cap {MAX_TAR_ENTRY_BYTES} in pack {}/{}",
                    d.name, d.version
                )));
            }
            total_bytes = total_bytes.saturating_add(declared_size);
            if total_bytes > MAX_TAR_TOTAL_BYTES {
                return Err(FetchError::Tar(format!(
                    "archive_too_large: total {total_bytes} > cap {MAX_TAR_TOTAL_BYTES} in pack {}/{}",
                    d.name, d.version
                )));
            }
            let path_buf = entry
                .path()
                .map_err(|e| FetchError::Tar(format!("path:{e}")))?
                .to_path_buf();
            let name = path_buf.to_string_lossy().to_string();

            // Reject any path that could escape the pack_dir.
            if path_buf.is_absolute()
                || name.contains('\\')
                || path_buf
                    .components()
                    .any(|c| matches!(c, std::path::Component::ParentDir | std::path::Component::Prefix(_) | std::path::Component::RootDir))
            {
                return Err(FetchError::Tar(format!(
                    "unsafe_tar_member: rejected path-traversal entry {:?} in pack {}/{}",
                    name, d.name, d.version
                )));
            }

            // Belt-and-braces: the header could lie about size, but tar's
            // `Entry::take()` bounds the actual read to the declared size.
            // We additionally guard the buffer capacity to avoid a
            // pathological reserve.
            let mut buf: Vec<u8> = Vec::with_capacity(
                (declared_size.min(MAX_TAR_ENTRY_BYTES)) as usize,
            );
            entry
                .read_to_end(&mut buf)
                .map_err(|e| FetchError::Tar(format!("read:{e}")))?;
            if (buf.len() as u64) > MAX_TAR_ENTRY_BYTES {
                return Err(FetchError::Tar(format!(
                    "entry_overflow_on_read: {} bytes > cap {} in pack {}/{}",
                    buf.len(), MAX_TAR_ENTRY_BYTES, d.name, d.version
                )));
            }
            let dst = pack_dir.join(&path_buf);

            // Defence in depth: the joined path must still start with pack_dir.
            // (Symlink-target escape is also blocked: we only ever write to
            // the joined path, never follow links.)
            if !dst.starts_with(&pack_dir) {
                return Err(FetchError::Tar(format!(
                    "unsafe_tar_member: joined path {:?} escapes pack_dir {:?}",
                    dst, pack_dir
                )));
            }

            if let Some(parent) = dst.parent() {
                fs::create_dir_all(parent)
                    .map_err(|e| FetchError::Io(format!("mkdir_extract:{e}")))?;
            }
            fs::write(&dst, &buf).map_err(|e| FetchError::Io(format!("write_extract:{e}")))?;
            if name == "manifest.json" {
                manifest = serde_json::from_slice(&buf)
                    .map_err(|e| FetchError::Json(format!("manifest:{e}")))?;
            } else if name.ends_with(".rego") {
                rego_files.push(name.clone());
            }
        }
    }

    tracing::info!(
        "pack_verified pack={}/{} sha256={} rego_files={}",
        d.name,
        d.version,
        got,
        rego_files.len()
    );

    Ok(RemotePack {
        name: d.name.clone(),
        version: d.version.clone(),
        sha256: got,
        tarball_path: tar_path,
        manifest,
        rego_files,
        signature_verified: true,
        inventory_only: true, // Rego v1 not executable by the subset evaluator.
    })
}

async fn fetch_bytes(client: &reqwest::Client, url: &str) -> Result<Vec<u8>, FetchError> {
    let resp = client
        .get(url)
        .send()
        .await
        .map_err(|e| FetchError::Http(format!("get:{e}")))?;
    let status = resp.status();
    if !status.is_success() {
        return Err(FetchError::Http(format!("status:{status} url={url}")));
    }
    resp.bytes()
        .await
        .map(|b| b.to_vec())
        .map_err(|e| FetchError::Http(format!("read:{e}")))
}

/// Render a 16-char fingerprint of all loaded packs' sha256s, in pack-load
/// order. Lets e2e tests assert on a stable identity for the pack-source.
pub fn fingerprint(packs: &[RemotePack]) -> String {
    let mut hasher = Sha256::new();
    for p in packs {
        hasher.update(p.name.as_bytes());
        hasher.update(b"@");
        hasher.update(p.version.as_bytes());
        hasher.update(b":");
        hasher.update(p.sha256.as_bytes());
        hasher.update(b"\n");
    }
    let digest = hasher.finalize();
    hex::encode(&digest[..8])
}