feat: add raw artifact audit trail

2026-06-24 09:38:15 +08:00
parent 6a17c9a336
commit a770070b34
12 changed files with 670 additions and 16 deletions
--- a/apps/api/migrations/202606240001_raw_artifacts.sql
+++ b/apps/api/migrations/202606240001_raw_artifacts.sql
@@ -0,0 +1,24 @@
+CREATE TABLE IF NOT EXISTS audit.raw_artifacts (
+    artifact_id BIGSERIAL PRIMARY KEY,
+    run_id BIGINT REFERENCES audit.ingestion_runs(run_id),
+    source_id BIGINT REFERENCES audit.data_sources(source_id),
+    original_filename TEXT NOT NULL,
+    raw_uri TEXT NOT NULL,
+    sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'),
+    size_bytes BIGINT NOT NULL CHECK (size_bytes >= 0),
+    mime_type TEXT,
+    uploaded_by TEXT NOT NULL DEFAULT 'system',
+    notes TEXT NOT NULL DEFAULT '',
+    created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+    UNIQUE (sha256)
+);
+
+ALTER TABLE audit.ingestion_runs
+    ADD COLUMN IF NOT EXISTS raw_artifact_id BIGINT REFERENCES audit.raw_artifacts(artifact_id);
+
+CREATE INDEX IF NOT EXISTS idx_raw_artifacts_run_id
+    ON audit.raw_artifacts(run_id);
+CREATE INDEX IF NOT EXISTS idx_raw_artifacts_source_id
+    ON audit.raw_artifacts(source_id);
+CREATE INDEX IF NOT EXISTS idx_raw_artifacts_created_at
+    ON audit.raw_artifacts(created_at DESC);
--- a/apps/api/src/handlers.rs
+++ b/apps/api/src/handlers.rs
@@ -5,10 +5,11 @@ use serde::Serialize;
 use crate::error::{ApiError, ApiResult};
 use crate::import_templates::validate_import_payload;
 use crate::models::{
-    AreaScore, AreaScoreSummary, CreateDataSource, CreateIngestionRun, CreateWatchlistItem,
-    DataSource, FinishIngestionRun, ImportValidationRequest, ImportValidationResponse,
-    IngestionRun, IngestionRunQuery, MarketOverview, MonthQuery, Neighborhood, NeighborhoodQuery,
-    NeighborhoodScore, NeighborhoodScoreQuery, UpdateWatchlistItem, WatchlistItem,
+    AreaScore, AreaScoreSummary, CreateDataSource, CreateIngestionRun, CreateRawArtifact,
+    CreateWatchlistItem, DataSource, FinishIngestionRun, ImportValidationRequest,
+    ImportValidationResponse, IngestionRun, IngestionRunQuery, MarketOverview, MonthQuery,
+    Neighborhood, NeighborhoodQuery, NeighborhoodScore, NeighborhoodScoreQuery, RawArtifact,
+    RawArtifactQuery, UpdateWatchlistItem, WatchlistItem,
 };
 use crate::state::AppState;

@@ -265,6 +266,7 @@ pub async fn list_ingestion_runs(
            r.run_id,
            r.source_id,
            s.name AS source_name,
+            r.raw_artifact_id,
            r.status,
            r.started_at::text AS started_at,
            r.finished_at::text AS finished_at,
@@ -299,10 +301,11 @@ pub async fn create_ingestion_run(
            source_id,
            status,
            raw_uri,
+            raw_artifact_id,
            row_count,
            error_message
        )
-        VALUES ($1, COALESCE($2, 'running'), $3, $4, $5)
+        VALUES ($1, COALESCE($2, 'running'), $3, $4, $5, $6)
        RETURNING
            run_id,
            source_id,
@@ -311,6 +314,7 @@ pub async fn create_ingestion_run(
                FROM audit.data_sources
                WHERE source_id = audit.ingestion_runs.source_id
            ) AS source_name,
+            raw_artifact_id,
            status,
            started_at::text AS started_at,
            finished_at::text AS finished_at,
@@ -322,6 +326,7 @@ pub async fn create_ingestion_run(
    .bind(payload.source_id)
    .bind(payload.status)
    .bind(payload.raw_uri)
+    .bind(payload.raw_artifact_id)
    .bind(payload.row_count)
    .bind(payload.error_message)
    .fetch_one(&state.pool)
@@ -355,6 +360,7 @@ pub async fn finish_ingestion_run(
                FROM audit.data_sources
                WHERE source_id = audit.ingestion_runs.source_id
            ) AS source_name,
+            raw_artifact_id,
            status,
            started_at::text AS started_at,
            finished_at::text AS finished_at,
@@ -374,6 +380,154 @@ pub async fn finish_ingestion_run(
    Ok(Json(run))
 }

+pub async fn list_raw_artifacts(
+    State(state): State<AppState>,
+    Query(query): Query<RawArtifactQuery>,
+) -> ApiResult<Json<Vec<RawArtifact>>> {
+    if let Some(sha256) = query.sha256.as_deref() {
+        validate_sha256(sha256)?;
+    }
+
+    let artifacts = sqlx::query_as::<_, RawArtifact>(
+        r#"
+        SELECT
+            a.artifact_id,
+            a.run_id,
+            a.source_id,
+            s.name AS source_name,
+            a.original_filename,
+            a.raw_uri,
+            a.sha256,
+            a.size_bytes,
+            a.mime_type,
+            a.uploaded_by,
+            a.notes,
+            a.created_at::text AS created_at
+        FROM audit.raw_artifacts a
+        LEFT JOIN audit.data_sources s ON s.source_id = a.source_id
+        WHERE ($1::bigint IS NULL OR a.source_id = $1)
+          AND (
+              $2::bigint IS NULL
+              OR a.run_id = $2
+              OR EXISTS (
+                  SELECT 1
+                  FROM audit.ingestion_runs r
+                  WHERE r.run_id = $2
+                    AND r.raw_artifact_id = a.artifact_id
+              )
+          )
+          AND ($3::text IS NULL OR a.sha256 = $3)
+        ORDER BY a.created_at DESC, a.artifact_id DESC
+        LIMIT 100
+        "#,
+    )
+    .bind(query.source_id)
+    .bind(query.run_id)
+    .bind(query.sha256)
+    .fetch_all(&state.pool)
+    .await?;
+
+    Ok(Json(artifacts))
+}
+
+pub async fn create_raw_artifact(
+    State(state): State<AppState>,
+    Json(payload): Json<CreateRawArtifact>,
+) -> ApiResult<Json<RawArtifact>> {
+    payload
+        .validate()
+        .map_err(|message| ApiError::BadRequest(message.to_string()))?;
+
+    let mut tx = state.pool.begin().await?;
+    let artifact = sqlx::query_as::<_, RawArtifact>(
+        r#"
+        INSERT INTO audit.raw_artifacts (
+            run_id,
+            source_id,
+            original_filename,
+            raw_uri,
+            sha256,
+            size_bytes,
+            mime_type,
+            uploaded_by,
+            notes
+        )
+        VALUES (
+            $1,
+            COALESCE(
+                $2,
+                (
+                    SELECT source_id
+                    FROM audit.ingestion_runs
+                    WHERE run_id = $1
+                )
+            ),
+            $3,
+            $4,
+            $5,
+            $6,
+            $7,
+            COALESCE($8, 'system'),
+            COALESCE($9, '')
+        )
+        ON CONFLICT (sha256) DO UPDATE
+        SET run_id = COALESCE(audit.raw_artifacts.run_id, EXCLUDED.run_id),
+            source_id = COALESCE(audit.raw_artifacts.source_id, EXCLUDED.source_id),
+            notes = CASE
+                WHEN audit.raw_artifacts.notes = '' THEN EXCLUDED.notes
+                ELSE audit.raw_artifacts.notes
+            END
+        RETURNING
+            artifact_id,
+            run_id,
+            source_id,
+            (
+                SELECT name
+                FROM audit.data_sources
+                WHERE source_id = audit.raw_artifacts.source_id
+            ) AS source_name,
+            original_filename,
+            raw_uri,
+            sha256,
+            size_bytes,
+            mime_type,
+            uploaded_by,
+            notes,
+            created_at::text AS created_at
+        "#,
+    )
+    .bind(payload.run_id)
+    .bind(payload.source_id)
+    .bind(payload.original_filename.trim().to_string())
+    .bind(payload.raw_uri.trim().to_string())
+    .bind(payload.sha256.trim().to_string())
+    .bind(payload.size_bytes)
+    .bind(payload.mime_type)
+    .bind(payload.uploaded_by)
+    .bind(payload.notes)
+    .fetch_one(&mut *tx)
+    .await?;
+
+    if let Some(run_id) = payload.run_id {
+        sqlx::query(
+            r#"
+            UPDATE audit.ingestion_runs
+            SET raw_artifact_id = $2,
+                raw_uri = COALESCE(raw_uri, $3)
+            WHERE run_id = $1
+            "#,
+        )
+        .bind(run_id)
+        .bind(artifact.artifact_id)
+        .bind(&artifact.raw_uri)
+        .execute(&mut *tx)
+        .await?;
+    }
+
+    tx.commit().await?;
+    Ok(Json(artifact))
+}
+
 pub async fn list_watchlist_items(
    State(state): State<AppState>,
 ) -> ApiResult<Json<Vec<WatchlistItem>>> {
@@ -563,14 +717,29 @@ fn validate_month(month: &str) -> ApiResult<()> {
    }
 }

+fn validate_sha256(value: &str) -> ApiResult<()> {
+    let valid = value.len() == 64
+        && value
+            .chars()
+            .all(|item| item.is_ascii_hexdigit() && !item.is_ascii_uppercase());
+
+    if valid {
+        Ok(())
+    } else {
+        Err(ApiError::BadRequest(
+            "sha256 must be a lowercase 64-character hex digest".to_string(),
+        ))
+    }
+}
+
 #[cfg(test)]
 mod tests {
    use crate::models::{
-        CreateDataSource, CreateIngestionRun, CreateWatchlistItem, FinishIngestionRun,
-        UpdateWatchlistItem,
+        CreateDataSource, CreateIngestionRun, CreateRawArtifact, CreateWatchlistItem,
+        FinishIngestionRun, UpdateWatchlistItem,
    };

-    use super::validate_month;
+    use super::{validate_month, validate_sha256};

    #[test]
    fn accepts_valid_month() {
@@ -674,6 +843,7 @@ mod tests {
            source_id: Some(1),
            status: Some("running".to_string()),
            raw_uri: Some("raw/official/2026-05.csv".to_string()),
+            raw_artifact_id: None,
            row_count: Some(10),
            error_message: None,
        }
@@ -684,6 +854,7 @@ mod tests {
            source_id: Some(1),
            status: Some("unknown".to_string()),
            raw_uri: None,
+            raw_artifact_id: None,
            row_count: None,
            error_message: None,
        }
@@ -706,4 +877,42 @@ mod tests {
        .validate()
        .is_err());
    }
+
+    #[test]
+    fn validates_raw_artifact_payload() {
+        assert!(CreateRawArtifact {
+            run_id: None,
+            source_id: None,
+            original_filename: "area.csv".to_string(),
+            raw_uri: "raw://sha256/area.csv".to_string(),
+            sha256: "a".repeat(64),
+            size_bytes: 128,
+            mime_type: Some("text/csv".to_string()),
+            uploaded_by: Some("researcher".to_string()),
+            notes: None,
+        }
+        .validate()
+        .is_ok());
+
+        assert!(CreateRawArtifact {
+            run_id: None,
+            source_id: None,
+            original_filename: "area.csv".to_string(),
+            raw_uri: "raw://sha256/area.csv".to_string(),
+            sha256: "A".repeat(64),
+            size_bytes: 128,
+            mime_type: None,
+            uploaded_by: None,
+            notes: None,
+        }
+        .validate()
+        .is_err());
+    }
+
+    #[test]
+    fn validates_sha256_query_value() {
+        assert!(validate_sha256(&"0".repeat(64)).is_ok());
+        assert!(validate_sha256(&"G".repeat(64)).is_err());
+        assert!(validate_sha256("short").is_err());
+    }
 }
--- a/apps/api/src/models.rs
+++ b/apps/api/src/models.rs
@@ -25,6 +25,13 @@ pub struct IngestionRunQuery {
    pub source_id: Option<i64>,
 }

+#[derive(Debug, Deserialize)]
+pub struct RawArtifactQuery {
+    pub source_id: Option<i64>,
+    pub run_id: Option<i64>,
+    pub sha256: Option<String>,
+}
+
 #[derive(Debug, Deserialize)]
 pub struct ImportValidationRequest {
    pub template_name: String,
@@ -211,6 +218,7 @@ pub struct IngestionRun {
    pub run_id: i64,
    pub source_id: Option<i64>,
    pub source_name: Option<String>,
+    pub raw_artifact_id: Option<i64>,
    pub status: String,
    pub started_at: String,
    pub finished_at: Option<String>,
@@ -224,6 +232,7 @@ pub struct CreateIngestionRun {
    pub source_id: Option<i64>,
    pub status: Option<String>,
    pub raw_uri: Option<String>,
+    pub raw_artifact_id: Option<i64>,
    pub row_count: Option<i32>,
    pub error_message: Option<String>,
 }
@@ -250,6 +259,67 @@ impl CreateIngestionRun {
    }
 }

+#[derive(Debug, Clone, Serialize, FromRow)]
+pub struct RawArtifact {
+    pub artifact_id: i64,
+    pub run_id: Option<i64>,
+    pub source_id: Option<i64>,
+    pub source_name: Option<String>,
+    pub original_filename: String,
+    pub raw_uri: String,
+    pub sha256: String,
+    pub size_bytes: i64,
+    pub mime_type: Option<String>,
+    pub uploaded_by: String,
+    pub notes: String,
+    pub created_at: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct CreateRawArtifact {
+    pub run_id: Option<i64>,
+    pub source_id: Option<i64>,
+    pub original_filename: String,
+    pub raw_uri: String,
+    pub sha256: String,
+    pub size_bytes: i64,
+    pub mime_type: Option<String>,
+    pub uploaded_by: Option<String>,
+    pub notes: Option<String>,
+}
+
+impl CreateRawArtifact {
+    pub fn validate(&self) -> Result<(), &'static str> {
+        if self.original_filename.trim().is_empty() {
+            return Err("original_filename is required");
+        }
+        if self.raw_uri.trim().is_empty() {
+            return Err("raw_uri is required");
+        }
+        if !is_valid_sha256(&self.sha256) {
+            return Err("sha256 must be a lowercase 64-character hex digest");
+        }
+        if self.size_bytes < 0 {
+            return Err("size_bytes must be non-negative");
+        }
+        for value in [
+            Some(self.original_filename.as_str()),
+            Some(self.raw_uri.as_str()),
+            self.mime_type.as_deref(),
+            self.uploaded_by.as_deref(),
+            self.notes.as_deref(),
+        ]
+        .into_iter()
+        .flatten()
+        {
+            if has_sensitive_text(value) {
+                return Err("payload contains sensitive text");
+            }
+        }
+        Ok(())
+    }
+}
+
 #[derive(Debug, Deserialize)]
 pub struct FinishIngestionRun {
    pub status: String,
@@ -319,6 +389,13 @@ fn has_sensitive_text(value: &str) -> bool {
        || value.starts_with("/private/")
 }

+fn is_valid_sha256(value: &str) -> bool {
+    value.len() == 64
+        && value
+            .chars()
+            .all(|item| item.is_ascii_hexdigit() && !item.is_ascii_uppercase())
+}
+
 #[derive(Debug, Deserialize)]
 pub struct UpdateWatchlistItem {
    pub target_price_psm: Option<f64>,
--- a/apps/api/src/routes.rs
+++ b/apps/api/src/routes.rs
@@ -4,10 +4,11 @@ use tower_http::cors::CorsLayer;
 use tower_http::trace::TraceLayer;

 use crate::handlers::{
-    archive_watchlist_item, create_data_source, create_ingestion_run, create_watchlist_item,
-    finish_ingestion_run, get_neighborhood, health, list_area_scores, list_data_sources,
-    list_ingestion_runs, list_neighborhood_scores, list_neighborhoods, list_watchlist_items,
-    market_overview, ready, update_watchlist_item, validate_import,
+    archive_watchlist_item, create_data_source, create_ingestion_run, create_raw_artifact,
+    create_watchlist_item, finish_ingestion_run, get_neighborhood, health, list_area_scores,
+    list_data_sources, list_ingestion_runs, list_neighborhood_scores, list_neighborhoods,
+    list_raw_artifacts, list_watchlist_items, market_overview, ready, update_watchlist_item,
+    validate_import,
 };
 use crate::state::AppState;

@@ -31,6 +32,10 @@ pub fn build_router(state: AppState) -> Router {
                    "/ingestion-runs/{run_id}/finish",
                    patch(finish_ingestion_run),
                )
+                .route(
+                    "/raw-artifacts",
+                    get(list_raw_artifacts).post(create_raw_artifact),
+                )
                .route("/imports/validate", axum::routing::post(validate_import))
                .route("/market/overview", get(market_overview))
                .route("/neighborhoods", get(list_neighborhoods))