mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
oxoxDev
@@ -11,6 +11,7 @@ pub mod firejail;
|
||||
pub mod landlock;
|
||||
pub mod live_policy;
|
||||
pub mod pairing;
|
||||
pub mod pii;
|
||||
pub mod policy;
|
||||
pub mod secrets;
|
||||
pub mod traits;
|
||||
@@ -50,3 +51,6 @@ pub use schemas::{
|
||||
all_controller_schemas as all_security_controller_schemas,
|
||||
all_registered_controllers as all_security_registered_controllers,
|
||||
};
|
||||
|
||||
#[allow(unused_imports)]
|
||||
pub use pii::{scan as scan_pii, CategoryHit, PiiCategory, PiiScanResult, RiskLevel};
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
//! Classifier entry point for the local PII detector.
|
||||
//!
|
||||
//! [`scan`] is a pure, synchronous function: it walks the compiled rule set
|
||||
//! (see [`super::rules`]) over the input, tallies matches per category, and
|
||||
//! folds the tally into a [`PiiScanResult`]. No async, no I/O, no network.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use super::rules::rules;
|
||||
use super::types::{CategoryHit, PiiCategory, PiiScanResult, RiskLevel};
|
||||
|
||||
/// Extra points added per additional co-occurring category. Multiple distinct
|
||||
/// identifier/context categories in one blob look more like a real record than
|
||||
/// any single one alone, so co-occurrence nudges the score upward.
|
||||
const CO_OCCURRENCE_BONUS: u32 = 5;
|
||||
|
||||
/// Map a raw score to a [`RiskLevel`]. Thresholds lean toward escalation
|
||||
/// (recall over precision).
|
||||
fn level_from_score(score: u32) -> RiskLevel {
|
||||
match score {
|
||||
0 => RiskLevel::None,
|
||||
1..=19 => RiskLevel::Low,
|
||||
20..=44 => RiskLevel::Medium,
|
||||
_ => RiskLevel::High,
|
||||
}
|
||||
}
|
||||
|
||||
/// Scan `content` for identification risk.
|
||||
///
|
||||
/// Returns a [`PiiScanResult`] carrying the overall [`RiskLevel`], a numeric
|
||||
/// score, and the distinct categories detected (with per-category counts).
|
||||
///
|
||||
/// Detection runs **fully locally** — pattern + keyword matching only. The
|
||||
/// function is deterministic and side-effect-free, so it is safe to call from
|
||||
/// any context (sync or async, on or off a Tokio runtime).
|
||||
///
|
||||
/// ## Scoring
|
||||
/// * Each distinct category contributes its [`PiiCategory::weight`] once.
|
||||
/// * Each additional co-occurring category adds [`CO_OCCURRENCE_BONUS`].
|
||||
/// * A "strong identifier" (SSN, card, passport, bank account) forces
|
||||
/// [`RiskLevel::High`] regardless of the numeric score.
|
||||
pub fn scan(content: &str) -> PiiScanResult {
|
||||
// BTreeMap keeps the output deterministic (categories emitted in enum
|
||||
// order) without a separate sort pass.
|
||||
let mut counts: BTreeMap<PiiCategory, usize> = BTreeMap::new();
|
||||
|
||||
for rule in rules() {
|
||||
for m in rule.regex.find_iter(content) {
|
||||
if let Some(validator) = rule.validator {
|
||||
if !validator(m.as_str()) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
*counts.entry(rule.category).or_default() += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if counts.is_empty() {
|
||||
log::trace!("[security][pii] scan clean (no categories matched)");
|
||||
return PiiScanResult::default();
|
||||
}
|
||||
|
||||
let distinct = counts.len() as u32;
|
||||
let mut score: u32 = counts.keys().map(|c| c.weight()).sum();
|
||||
if distinct > 1 {
|
||||
score += CO_OCCURRENCE_BONUS * (distinct - 1);
|
||||
}
|
||||
|
||||
let has_strong = counts.keys().any(|c| c.is_strong_identifier());
|
||||
let mut level = level_from_score(score);
|
||||
if has_strong && level < RiskLevel::High {
|
||||
level = RiskLevel::High;
|
||||
}
|
||||
|
||||
let categories: Vec<PiiCategory> = counts.keys().copied().collect();
|
||||
let hits: Vec<CategoryHit> = counts
|
||||
.iter()
|
||||
.map(|(&category, &count)| CategoryHit { category, count })
|
||||
.collect();
|
||||
|
||||
// Log level/score/categories only — never the matched content itself.
|
||||
log::debug!(
|
||||
"[security][pii] scan level={} score={} categories={:?}",
|
||||
level.as_str(),
|
||||
score,
|
||||
categories.iter().map(|c| c.as_str()).collect::<Vec<_>>()
|
||||
);
|
||||
|
||||
PiiScanResult {
|
||||
level,
|
||||
score,
|
||||
categories,
|
||||
hits,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn empty_input_is_none() {
|
||||
let r = scan("");
|
||||
assert_eq!(r.level, RiskLevel::None);
|
||||
assert_eq!(r.score, 0);
|
||||
assert!(r.categories.is_empty());
|
||||
assert!(!r.is_sensitive());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn level_thresholds_are_monotonic() {
|
||||
assert_eq!(level_from_score(0), RiskLevel::None);
|
||||
assert_eq!(level_from_score(1), RiskLevel::Low);
|
||||
assert_eq!(level_from_score(19), RiskLevel::Low);
|
||||
assert_eq!(level_from_score(20), RiskLevel::Medium);
|
||||
assert_eq!(level_from_score(44), RiskLevel::Medium);
|
||||
assert_eq!(level_from_score(45), RiskLevel::High);
|
||||
assert_eq!(level_from_score(1000), RiskLevel::High);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
//! Local PII / identification-risk detector (privacy epic #4256, slice S5).
|
||||
//!
|
||||
//! Detects when a prompt or document may contain patient / legal / financial /
|
||||
//! family or other identifiable information, and reports a risk **level**, a
|
||||
//! numeric **score**, and the matched **categories** — so upstream gates can
|
||||
//! decide whether content is safe to send off-device.
|
||||
//!
|
||||
//! ## Hard guarantee: detection is fully local
|
||||
//! Classification is pure pattern + keyword matching over the input string.
|
||||
//! There is **no** network call, no async, no external model — importing this
|
||||
//! module pulls in nothing beyond `regex`. Sending content to a remote service
|
||||
//! to classify it would defeat the entire purpose of the privacy epic, so the
|
||||
//! shipped default is dependency-free and offline by construction.
|
||||
//!
|
||||
//! ## Recall over precision
|
||||
//! A missed flag means sensitive data leaves the machine unnoticed, so the
|
||||
//! detector is tuned to over-flag rather than under-flag. See [`detector::scan`]
|
||||
//! for the scoring model.
|
||||
//!
|
||||
//! ```
|
||||
//! use openhuman_core::openhuman::security::pii::{scan, PiiCategory, RiskLevel};
|
||||
//!
|
||||
//! let result = scan("patient John was diagnosed; SSN 123-45-6789");
|
||||
//! assert_eq!(result.level, RiskLevel::High);
|
||||
//! assert!(result.has_category(PiiCategory::NationalId));
|
||||
//! ```
|
||||
|
||||
mod detector;
|
||||
mod rules;
|
||||
mod types;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub use detector::scan;
|
||||
pub use types::{CategoryHit, PiiCategory, PiiScanResult, RiskLevel};
|
||||
@@ -0,0 +1,250 @@
|
||||
//! Detection rules for the local PII detector.
|
||||
//!
|
||||
//! Two flavours of rule, both compiled once into a process-wide static:
|
||||
//!
|
||||
//! * **Pattern rules** — a [`Regex`] plus an optional structural `validator`
|
||||
//! (Luhn for cards, octet range for IPv4) that rejects shape-matches that
|
||||
//! aren't actually valid identifiers.
|
||||
//! * **Keyword rules** — an alternation of domain terms wrapped in word
|
||||
//! boundaries, used for the topical categories (medical / legal / financial /
|
||||
//! family) where there is no single canonical value shape.
|
||||
//!
|
||||
//! All matching is case-insensitive and offline; nothing here performs I/O.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
use super::types::PiiCategory;
|
||||
|
||||
/// A single detection rule: matches [`regex`](Rule::regex), attributes hits to
|
||||
/// [`category`](Rule::category), and — if present — only counts a match when
|
||||
/// [`validator`](Rule::validator) confirms it.
|
||||
pub(crate) struct Rule {
|
||||
pub category: PiiCategory,
|
||||
pub regex: Regex,
|
||||
/// Optional structural check applied to each raw match before it counts.
|
||||
pub validator: Option<fn(&str) -> bool>,
|
||||
}
|
||||
|
||||
/// Build a keyword rule: a case-insensitive, word-boundaried alternation over
|
||||
/// `terms`, all attributed to `category`.
|
||||
fn keyword_rule(category: PiiCategory, terms: &[&str]) -> Rule {
|
||||
let alternation = terms.join("|");
|
||||
let pattern = format!(r"(?i)\b(?:{alternation})\b");
|
||||
Rule {
|
||||
category,
|
||||
regex: Regex::new(&pattern).expect("keyword rule regex is well-formed"),
|
||||
validator: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build a pattern rule from a raw regex string.
|
||||
fn pattern_rule(category: PiiCategory, pattern: &str, validator: Option<fn(&str) -> bool>) -> Rule {
|
||||
Rule {
|
||||
category,
|
||||
regex: Regex::new(pattern).expect("pattern rule regex is well-formed"),
|
||||
validator,
|
||||
}
|
||||
}
|
||||
|
||||
/// Luhn checksum validation for candidate payment-card numbers. Strips
|
||||
/// separators first; requires 13–19 digits.
|
||||
pub(crate) fn is_luhn_valid(raw: &str) -> bool {
|
||||
let digits: Vec<u8> = raw
|
||||
.bytes()
|
||||
.filter(|b| b.is_ascii_digit())
|
||||
.map(|b| b - b'0')
|
||||
.collect();
|
||||
if !(13..=19).contains(&digits.len()) {
|
||||
return false;
|
||||
}
|
||||
let mut sum = 0u32;
|
||||
// Double every second digit from the right.
|
||||
for (i, &d) in digits.iter().rev().enumerate() {
|
||||
let mut v = d as u32;
|
||||
if i % 2 == 1 {
|
||||
v *= 2;
|
||||
if v > 9 {
|
||||
v -= 9;
|
||||
}
|
||||
}
|
||||
sum += v;
|
||||
}
|
||||
sum % 10 == 0
|
||||
}
|
||||
|
||||
/// Validate that a dotted-quad match is a real IPv4 address (each octet 0–255).
|
||||
pub(crate) fn is_ipv4(raw: &str) -> bool {
|
||||
let octets: Vec<&str> = raw.split('.').collect();
|
||||
octets.len() == 4
|
||||
&& octets
|
||||
.iter()
|
||||
.all(|o| !o.is_empty() && o.parse::<u32>().is_ok_and(|n| n <= 255))
|
||||
}
|
||||
|
||||
/// The full, process-wide rule set. Compiled lazily on first [`scan`] call and
|
||||
/// reused for the life of the process.
|
||||
///
|
||||
/// [`scan`]: super::detector::scan
|
||||
pub(crate) fn rules() -> &'static [Rule] {
|
||||
static RULES: LazyLock<Vec<Rule>> = LazyLock::new(build_rules);
|
||||
&RULES
|
||||
}
|
||||
|
||||
fn build_rules() -> Vec<Rule> {
|
||||
vec![
|
||||
// --- structured identifiers -------------------------------------
|
||||
// Email.
|
||||
pattern_rule(
|
||||
PiiCategory::Email,
|
||||
r"\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}\b",
|
||||
None,
|
||||
),
|
||||
// US Social Security Number (3-2-4, dash or space separated). The
|
||||
// separator requirement keeps it distinct from phone numbers (3-3-4)
|
||||
// and from bare 9-digit blobs.
|
||||
pattern_rule(
|
||||
PiiCategory::NationalId,
|
||||
r"\b\d{3}[-\s]\d{2}[-\s]\d{4}\b",
|
||||
None,
|
||||
),
|
||||
// Payment card: 13–19 digits with optional single separators, then
|
||||
// Luhn-validated so version strings / IDs don't false-positive.
|
||||
pattern_rule(
|
||||
PiiCategory::CreditCard,
|
||||
r"\b\d(?:[ -]?\d){12,18}\b",
|
||||
Some(is_luhn_valid),
|
||||
),
|
||||
// IBAN bank account: 2-letter country + 2 check digits + 11–30 alnum.
|
||||
pattern_rule(
|
||||
PiiCategory::BankAccount,
|
||||
r"\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b",
|
||||
None,
|
||||
),
|
||||
// US-style phone number (3-3-4 groups, requires a separator so it
|
||||
// doesn't swallow arbitrary 10-digit numbers).
|
||||
pattern_rule(
|
||||
PiiCategory::PhoneNumber,
|
||||
r"\b(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]\d{3}[-.\s]\d{4}\b",
|
||||
None,
|
||||
),
|
||||
// International E.164-ish phone number (leading + and 7–15 digits).
|
||||
pattern_rule(PiiCategory::PhoneNumber, r"\+\d(?:[\d\s\-]{6,16}\d)", None),
|
||||
// IPv4 address (octet-validated).
|
||||
pattern_rule(
|
||||
PiiCategory::IpAddress,
|
||||
r"\b(?:\d{1,3}\.){3}\d{1,3}\b",
|
||||
Some(is_ipv4),
|
||||
),
|
||||
// Street / postal address (number + words + street-type keyword).
|
||||
pattern_rule(
|
||||
PiiCategory::PostalAddress,
|
||||
r"(?i)\b\d{1,6}\s+[A-Za-z0-9.'\- ]{2,40}?\b(?:street|st|avenue|ave|road|rd|boulevard|blvd|lane|ln|drive|dr|court|ct|way|place|pl|terrace|ter|circle|cir|highway|hwy)\b",
|
||||
None,
|
||||
),
|
||||
// Date of birth (explicit context — pattern-based, not a keyword list).
|
||||
pattern_rule(
|
||||
PiiCategory::DateOfBirth,
|
||||
r"(?i)\b(?:date of birth|d\.?o\.?b\.?|birth ?date|born on)\b",
|
||||
None,
|
||||
),
|
||||
// --- sensitive topical context (keyword-based) ------------------
|
||||
keyword_rule(PiiCategory::Passport, &["passport"]),
|
||||
keyword_rule(
|
||||
PiiCategory::Medical,
|
||||
&[
|
||||
"patient",
|
||||
"diagnosis",
|
||||
"diagnosed",
|
||||
"prescription",
|
||||
"prescribed",
|
||||
"symptom",
|
||||
"symptoms",
|
||||
"medication",
|
||||
"medications",
|
||||
"treatment",
|
||||
"physician",
|
||||
"hospital",
|
||||
"clinic",
|
||||
"disease",
|
||||
"medical record",
|
||||
"health record",
|
||||
"mrn",
|
||||
"blood pressure",
|
||||
"dosage",
|
||||
"therapy",
|
||||
"biopsy",
|
||||
"oncology",
|
||||
"cardiology",
|
||||
"psychiatric",
|
||||
"mental health",
|
||||
"allergy",
|
||||
"allergies",
|
||||
],
|
||||
),
|
||||
keyword_rule(
|
||||
PiiCategory::Legal,
|
||||
&[
|
||||
"plaintiff",
|
||||
"defendant",
|
||||
"attorney",
|
||||
"lawsuit",
|
||||
"litigation",
|
||||
"case number",
|
||||
"docket",
|
||||
"subpoena",
|
||||
"settlement",
|
||||
"indictment",
|
||||
"deposition",
|
||||
"legal counsel",
|
||||
"prosecution",
|
||||
"testimony",
|
||||
"affidavit",
|
||||
"arraignment",
|
||||
"felony",
|
||||
"misdemeanor",
|
||||
"custody",
|
||||
"court order",
|
||||
],
|
||||
),
|
||||
keyword_rule(
|
||||
PiiCategory::Financial,
|
||||
&[
|
||||
"bank account",
|
||||
"account number",
|
||||
"routing number",
|
||||
"salary",
|
||||
"annual income",
|
||||
"credit score",
|
||||
"tax id",
|
||||
"invoice",
|
||||
"sort code",
|
||||
"swift code",
|
||||
"wire transfer",
|
||||
"mortgage",
|
||||
"loan balance",
|
||||
"net worth",
|
||||
"iban",
|
||||
],
|
||||
),
|
||||
keyword_rule(
|
||||
PiiCategory::Family,
|
||||
&[
|
||||
"spouse",
|
||||
"wife",
|
||||
"husband",
|
||||
"daughter",
|
||||
"son",
|
||||
"children",
|
||||
"minor child",
|
||||
"maiden name",
|
||||
"next of kin",
|
||||
"dependent",
|
||||
"marital status",
|
||||
"guardian",
|
||||
"sibling",
|
||||
],
|
||||
),
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
//! Fixture + boundary tests for the local PII detector.
|
||||
//!
|
||||
//! Coverage intent:
|
||||
//! * every category detects its canonical form,
|
||||
//! * false-positive guards (benign text, invalid card/IP) do NOT flag,
|
||||
//! * false-negative / recall guards (varied formats, in-sentence values) DO
|
||||
//! flag — recall matters more than precision here,
|
||||
//! * risk-level escalation behaves as specified,
|
||||
//! * detection is deterministic + offline (no external egress on classify),
|
||||
//! * category identifiers serialize to stable snake_case strings.
|
||||
|
||||
use super::rules::{is_ipv4, is_luhn_valid};
|
||||
use super::{scan, PiiCategory, RiskLevel};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Per-category detection (recall / true positives)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn detects_email() {
|
||||
let r = scan("reach me at alice.smith@example.com any time");
|
||||
assert!(r.has_category(PiiCategory::Email));
|
||||
assert!(r.is_sensitive());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_national_id_ssn() {
|
||||
// Dash- and space-separated both count (recall).
|
||||
for raw in ["SSN 123-45-6789", "ssn is 123 45 6789"] {
|
||||
let r = scan(raw);
|
||||
assert!(r.has_category(PiiCategory::NationalId), "missed in {raw:?}");
|
||||
assert_eq!(r.level, RiskLevel::High, "strong id must be High: {raw:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_valid_credit_card() {
|
||||
// 4111 1111 1111 1111 is the canonical Visa test number (Luhn-valid).
|
||||
for raw in [
|
||||
"card 4111 1111 1111 1111 exp 12/26",
|
||||
"4111-1111-1111-1111",
|
||||
"4111111111111111",
|
||||
] {
|
||||
let r = scan(raw);
|
||||
assert!(r.has_category(PiiCategory::CreditCard), "missed in {raw:?}");
|
||||
assert_eq!(r.level, RiskLevel::High);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_iban_bank_account() {
|
||||
let r = scan("transfer to GB82WEST12345698765432 please");
|
||||
assert!(r.has_category(PiiCategory::BankAccount));
|
||||
assert_eq!(r.level, RiskLevel::High);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_phone_numbers_varied_formats() {
|
||||
for raw in [
|
||||
"call 415-555-1234",
|
||||
"(415) 555 1234",
|
||||
"+1 415.555.1234",
|
||||
"reach +44 20 7946 0958 in london",
|
||||
] {
|
||||
let r = scan(raw);
|
||||
assert!(
|
||||
r.has_category(PiiCategory::PhoneNumber),
|
||||
"missed in {raw:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_ipv4_address() {
|
||||
let r = scan("client connected from 192.168.10.24 last night");
|
||||
assert!(r.has_category(PiiCategory::IpAddress));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_postal_address() {
|
||||
let r = scan("ship to 1600 Pennsylvania Avenue, Washington");
|
||||
assert!(r.has_category(PiiCategory::PostalAddress));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_date_of_birth_context() {
|
||||
for raw in [
|
||||
"date of birth: 1990-04-12",
|
||||
"DOB 04/12/1990",
|
||||
"he was born on the 3rd",
|
||||
] {
|
||||
let r = scan(raw);
|
||||
assert!(
|
||||
r.has_category(PiiCategory::DateOfBirth),
|
||||
"missed in {raw:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_passport() {
|
||||
let r = scan("upload a scan of your passport");
|
||||
assert!(r.has_category(PiiCategory::Passport));
|
||||
assert_eq!(r.level, RiskLevel::High); // strong identifier
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_medical_context() {
|
||||
let r = scan("the patient was diagnosed and prescribed medication");
|
||||
assert!(r.has_category(PiiCategory::Medical));
|
||||
assert!(r.level >= RiskLevel::Medium);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_legal_context() {
|
||||
let r = scan("the plaintiff filed a lawsuit; the defendant's attorney responded");
|
||||
assert!(r.has_category(PiiCategory::Legal));
|
||||
assert!(r.level >= RiskLevel::Medium);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_financial_context() {
|
||||
let r = scan("please send your bank account and routing number");
|
||||
assert!(r.has_category(PiiCategory::Financial));
|
||||
assert!(r.level >= RiskLevel::Medium);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_family_context() {
|
||||
let r = scan("my spouse and our minor child are dependents");
|
||||
assert!(r.has_category(PiiCategory::Family));
|
||||
assert!(r.is_sensitive());
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// False-positive guards
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn benign_text_is_not_flagged() {
|
||||
for raw in [
|
||||
"let's grab coffee at 3pm tomorrow",
|
||||
"the build passed and the tests are green",
|
||||
"refactor the parser to be faster",
|
||||
"version 1.2.3 shipped yesterday", // dotted, but only 3 octets — not IPv4
|
||||
] {
|
||||
let r = scan(raw);
|
||||
assert_eq!(r.level, RiskLevel::None, "false positive on {raw:?}: {r:?}");
|
||||
assert!(r.categories.is_empty(), "categories on {raw:?}: {r:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn luhn_invalid_number_is_not_a_card() {
|
||||
// 16 digits but fails the Luhn checksum → must NOT be a CreditCard.
|
||||
let r = scan("order id 1234 5678 9012 3456 confirmed");
|
||||
assert!(
|
||||
!r.has_category(PiiCategory::CreditCard),
|
||||
"Luhn-invalid number flagged as card: {r:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn out_of_range_octets_are_not_ipv4() {
|
||||
let r = scan("coords 300.400.500.600 are nonsense");
|
||||
assert!(
|
||||
!r.has_category(PiiCategory::IpAddress),
|
||||
"bad octets flagged: {r:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn phone_shaped_ssn_not_double_counted_as_ssn() {
|
||||
// 3-3-4 grouping is a phone, not a 3-2-4 SSN.
|
||||
let r = scan("call 415-555-1234");
|
||||
assert!(r.has_category(PiiCategory::PhoneNumber));
|
||||
assert!(!r.has_category(PiiCategory::NationalId));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Risk-level escalation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn single_weak_identifier_is_low() {
|
||||
let r = scan("email me: bob@example.org");
|
||||
assert_eq!(r.level, RiskLevel::Low);
|
||||
assert_eq!(r.categories, vec![PiiCategory::Email]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn co_occurring_identifiers_escalate() {
|
||||
// Email + phone together → beyond a single weak signal.
|
||||
let r = scan("bob@example.org / 415-555-1234");
|
||||
assert!(r.has_category(PiiCategory::Email));
|
||||
assert!(r.has_category(PiiCategory::PhoneNumber));
|
||||
assert!(
|
||||
r.level >= RiskLevel::Medium,
|
||||
"expected escalation, got {r:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strong_identifier_forces_high_even_alone() {
|
||||
let r = scan("123-45-6789");
|
||||
assert_eq!(r.level, RiskLevel::High);
|
||||
assert!(r.score >= PiiCategory::NationalId.weight());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn medical_record_shape_is_high() {
|
||||
let r = scan("patient John Doe, DOB 01/02/1980, diagnosed; SSN 111-22-3333");
|
||||
assert_eq!(r.level, RiskLevel::High);
|
||||
assert!(r.has_category(PiiCategory::Medical));
|
||||
assert!(r.has_category(PiiCategory::NationalId));
|
||||
assert!(r.has_category(PiiCategory::DateOfBirth));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn score_increases_with_more_categories() {
|
||||
let one = scan("bob@example.org").score;
|
||||
let two = scan("bob@example.org and patient diagnosed").score;
|
||||
assert!(
|
||||
two > one,
|
||||
"adding a category should raise the score: {one} !< {two}"
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Offline / determinism (no external egress on classify)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn scan_is_deterministic_and_pure() {
|
||||
// Detection performs zero I/O: it is a pure function of its input, so
|
||||
// repeated calls must be byte-for-byte identical. This is the unit-test
|
||||
// proxy for "no external egress occurs during classify" — there is no
|
||||
// async, no network client, and nothing in this module to mock away.
|
||||
let input = "patient bob@example.org, card 4111 1111 1111 1111, SSN 123-45-6789";
|
||||
let first = scan(input);
|
||||
let second = scan(input);
|
||||
assert_eq!(first, second);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scan_runs_without_a_tokio_runtime() {
|
||||
// A plain `#[test]` has no Tokio runtime installed; the fact that `scan`
|
||||
// returns here at all proves it does not spawn async work / touch the
|
||||
// network to classify.
|
||||
let r = scan("SSN 123-45-6789");
|
||||
assert_eq!(r.level, RiskLevel::High);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Stable serde identifiers (contract for S2 / RPC / event bus)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn category_identifiers_are_stable_snake_case() {
|
||||
let cases = [
|
||||
(PiiCategory::Email, "email"),
|
||||
(PiiCategory::PhoneNumber, "phone_number"),
|
||||
(PiiCategory::NationalId, "national_id"),
|
||||
(PiiCategory::CreditCard, "credit_card"),
|
||||
(PiiCategory::BankAccount, "bank_account"),
|
||||
(PiiCategory::IpAddress, "ip_address"),
|
||||
(PiiCategory::PostalAddress, "postal_address"),
|
||||
(PiiCategory::DateOfBirth, "date_of_birth"),
|
||||
(PiiCategory::Passport, "passport"),
|
||||
(PiiCategory::Medical, "medical"),
|
||||
(PiiCategory::Legal, "legal"),
|
||||
(PiiCategory::Financial, "financial"),
|
||||
(PiiCategory::Family, "family"),
|
||||
];
|
||||
for (cat, expected) in cases {
|
||||
assert_eq!(cat.as_str(), expected);
|
||||
assert_eq!(
|
||||
serde_json::to_string(&cat).unwrap(),
|
||||
format!("\"{expected}\"")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn risk_levels_are_ordered_and_stable() {
|
||||
assert!(RiskLevel::None < RiskLevel::Low);
|
||||
assert!(RiskLevel::Low < RiskLevel::Medium);
|
||||
assert!(RiskLevel::Medium < RiskLevel::High);
|
||||
for (lvl, s) in [
|
||||
(RiskLevel::None, "none"),
|
||||
(RiskLevel::Low, "low"),
|
||||
(RiskLevel::Medium, "medium"),
|
||||
(RiskLevel::High, "high"),
|
||||
] {
|
||||
assert_eq!(lvl.as_str(), s);
|
||||
assert_eq!(serde_json::to_string(&lvl).unwrap(), format!("\"{s}\""));
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Validator unit tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn luhn_validator_accepts_known_good_and_rejects_bad() {
|
||||
assert!(is_luhn_valid("4111111111111111")); // Visa test number
|
||||
assert!(is_luhn_valid("4111 1111 1111 1111"));
|
||||
assert!(!is_luhn_valid("1234567890123456"));
|
||||
assert!(!is_luhn_valid("12345")); // too short
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ipv4_validator_bounds_octets() {
|
||||
assert!(is_ipv4("192.168.0.1"));
|
||||
assert!(is_ipv4("255.255.255.255"));
|
||||
assert!(!is_ipv4("256.1.1.1"));
|
||||
assert!(!is_ipv4("1.2.3"));
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
//! Result + category types for the local PII / identification-risk detector
|
||||
//! (privacy epic #4256, slice S5).
|
||||
//!
|
||||
//! ## Privacy note
|
||||
//! The detector deliberately reports **categories and counts only** — never
|
||||
//! the raw matched substrings. The whole point of S5 is to reason about
|
||||
//! identification risk *without* copying the sensitive value anywhere, so the
|
||||
//! result type itself must not become a new PII sink (mirrors the "no response
|
||||
//! body in telemetry" rule the codebase already follows).
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Identification-risk level assigned to a scanned piece of content.
|
||||
///
|
||||
/// Ordered `None < Low < Medium < High` (declaration order drives the derived
|
||||
/// `Ord`), so callers can gate behaviour with a plain comparison such as
|
||||
/// `result.level >= RiskLevel::Medium`.
|
||||
///
|
||||
/// The detector is tuned toward **recall**: a false positive (flagging benign
|
||||
/// text) is far cheaper than a false negative (letting a real identifier leave
|
||||
/// the machine unflagged), so the thresholds lean toward escalation.
|
||||
#[derive(
|
||||
Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Serialize, Deserialize,
|
||||
)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum RiskLevel {
|
||||
/// No identification-risk signal found.
|
||||
#[default]
|
||||
None,
|
||||
/// A single weak signal (e.g. one email or phone number in isolation).
|
||||
Low,
|
||||
/// A moderate signal — sensitive topical context, or a couple of
|
||||
/// co-occurring identifiers.
|
||||
Medium,
|
||||
/// A strong signal — a high-confidence identifier (SSN, card, passport,
|
||||
/// bank account) or several co-occurring categories that together look
|
||||
/// like a real record.
|
||||
High,
|
||||
}
|
||||
|
||||
impl RiskLevel {
|
||||
/// Stable lowercase identifier (matches the serde representation).
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
RiskLevel::None => "none",
|
||||
RiskLevel::Low => "low",
|
||||
RiskLevel::Medium => "medium",
|
||||
RiskLevel::High => "high",
|
||||
}
|
||||
}
|
||||
|
||||
/// `true` when the content carries any identification risk at all
|
||||
/// (anything above [`RiskLevel::None`]). Downstream gates (S3 indicator,
|
||||
/// S4 sensitive-transfer gate) use this as the coarse "should we care?"
|
||||
/// signal.
|
||||
pub fn is_sensitive(self) -> bool {
|
||||
self > RiskLevel::None
|
||||
}
|
||||
}
|
||||
|
||||
/// A single identification-risk category the detector knows how to recognise.
|
||||
///
|
||||
/// The set covers both **structured identifiers** (pattern-matched: email,
|
||||
/// phone, national ID/SSN, card, bank account, IP, postal address) and
|
||||
/// **sensitive topical context** (keyword-matched: medical, legal, financial,
|
||||
/// family, date-of-birth, passport) — the category set called out in the S5
|
||||
/// scope (patient / legal / financial / family / personal identifiers).
|
||||
///
|
||||
/// Serde uses stable `snake_case` identifiers so the values are safe to carry
|
||||
/// across the RPC / event-bus boundary and into the S2 egress descriptor
|
||||
/// without a breaking reshape.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum PiiCategory {
|
||||
// --- structured identifiers (pattern-based) ---
|
||||
/// Email address (`alice@example.com`).
|
||||
Email,
|
||||
/// Telephone number (US or international formats).
|
||||
PhoneNumber,
|
||||
/// National identifier — US Social Security Number and similar
|
||||
/// dash/space-separated government IDs.
|
||||
NationalId,
|
||||
/// Payment card number (validated with the Luhn checksum).
|
||||
CreditCard,
|
||||
/// Bank account number in IBAN form.
|
||||
BankAccount,
|
||||
/// IPv4 address (octet-validated).
|
||||
IpAddress,
|
||||
/// Street / postal address.
|
||||
PostalAddress,
|
||||
// --- sensitive topical context (keyword-based) ---
|
||||
/// Date of birth (explicit "DOB" / "date of birth" context).
|
||||
DateOfBirth,
|
||||
/// Passport reference.
|
||||
Passport,
|
||||
/// Medical / patient / health information.
|
||||
Medical,
|
||||
/// Legal / litigation / case information.
|
||||
Legal,
|
||||
/// Financial / banking / income information.
|
||||
Financial,
|
||||
/// Family / next-of-kin / personal-relationship information.
|
||||
Family,
|
||||
}
|
||||
|
||||
impl PiiCategory {
|
||||
/// Stable lowercase identifier (matches the serde representation). Handy
|
||||
/// for logs and for downstream code that keys off the string form.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
PiiCategory::Email => "email",
|
||||
PiiCategory::PhoneNumber => "phone_number",
|
||||
PiiCategory::NationalId => "national_id",
|
||||
PiiCategory::CreditCard => "credit_card",
|
||||
PiiCategory::BankAccount => "bank_account",
|
||||
PiiCategory::IpAddress => "ip_address",
|
||||
PiiCategory::PostalAddress => "postal_address",
|
||||
PiiCategory::DateOfBirth => "date_of_birth",
|
||||
PiiCategory::Passport => "passport",
|
||||
PiiCategory::Medical => "medical",
|
||||
PiiCategory::Legal => "legal",
|
||||
PiiCategory::Financial => "financial",
|
||||
PiiCategory::Family => "family",
|
||||
}
|
||||
}
|
||||
|
||||
/// Scoring weight contributed when this category is present. Higher =
|
||||
/// stronger identification signal. Tuned toward recall — see
|
||||
/// [`crate::openhuman::security::pii::detector`] for how weights combine.
|
||||
pub(crate) fn weight(self) -> u32 {
|
||||
match self {
|
||||
// High-confidence direct identifiers — enough to flag on their own.
|
||||
PiiCategory::NationalId => 50,
|
||||
PiiCategory::CreditCard => 50,
|
||||
PiiCategory::Passport => 45,
|
||||
PiiCategory::BankAccount => 40,
|
||||
// Sensitive topical context.
|
||||
PiiCategory::Medical => 30,
|
||||
PiiCategory::Legal => 25,
|
||||
PiiCategory::Financial => 25,
|
||||
// Moderate identifiers.
|
||||
PiiCategory::DateOfBirth => 20,
|
||||
PiiCategory::PostalAddress => 20,
|
||||
PiiCategory::Family => 20,
|
||||
// Weak / commonly-shared identifiers.
|
||||
PiiCategory::Email => 15,
|
||||
PiiCategory::PhoneNumber => 15,
|
||||
PiiCategory::IpAddress => 10,
|
||||
}
|
||||
}
|
||||
|
||||
/// A "strong identifier" is a high-confidence, directly-identifying value
|
||||
/// whose mere presence should force [`RiskLevel::High`], independent of the
|
||||
/// numeric score. Guards recall against future weight edits.
|
||||
pub fn is_strong_identifier(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
PiiCategory::NationalId
|
||||
| PiiCategory::CreditCard
|
||||
| PiiCategory::Passport
|
||||
| PiiCategory::BankAccount
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-category match tally. Carries the category and how many times it was
|
||||
/// seen — never the matched text itself.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct CategoryHit {
|
||||
/// Which category matched.
|
||||
pub category: PiiCategory,
|
||||
/// Number of matches for this category across the scanned content.
|
||||
pub count: usize,
|
||||
}
|
||||
|
||||
/// Outcome of scanning a piece of content for identification risk.
|
||||
///
|
||||
/// This is the public contract S3/S4 (and, after rebase, the S2 egress
|
||||
/// descriptor) consume. It intentionally holds only the risk level, a numeric
|
||||
/// score, and the matched categories/counts — no raw values.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
pub struct PiiScanResult {
|
||||
/// Overall identification-risk level.
|
||||
pub level: RiskLevel,
|
||||
/// Numeric score behind [`level`](Self::level). Exposed for callers that
|
||||
/// want a finer-grained ranking than the four-way enum.
|
||||
pub score: u32,
|
||||
/// Distinct categories detected, sorted and de-duplicated for
|
||||
/// deterministic output.
|
||||
pub categories: Vec<PiiCategory>,
|
||||
/// Per-category match counts, sorted by category.
|
||||
pub hits: Vec<CategoryHit>,
|
||||
}
|
||||
|
||||
impl PiiScanResult {
|
||||
/// Convenience: content carries at least some identification risk.
|
||||
pub fn is_sensitive(&self) -> bool {
|
||||
self.level.is_sensitive()
|
||||
}
|
||||
|
||||
/// Convenience: was a specific category detected?
|
||||
pub fn has_category(&self, category: PiiCategory) -> bool {
|
||||
self.categories.contains(&category)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user