feat: add data source ingestion audit api

apps/api/src/handlers.rs

@@ -4,9 +4,10 @@ use serde::Serialize;
 
 use crate::error::{ApiError, ApiResult};
 use crate::models::{
-    AreaScore, AreaScoreSummary, CreateWatchlistItem, MarketOverview, MonthQuery, Neighborhood,
-    NeighborhoodQuery, NeighborhoodScore, NeighborhoodScoreQuery, UpdateWatchlistItem,
-    WatchlistItem,
+    AreaScore, AreaScoreSummary, CreateDataSource, CreateIngestionRun, CreateWatchlistItem,
+    DataSource, FinishIngestionRun, IngestionRun, IngestionRunQuery, MarketOverview, MonthQuery,
+    Neighborhood, NeighborhoodQuery, NeighborhoodScore, NeighborhoodScoreQuery,
+    UpdateWatchlistItem, WatchlistItem,
 };
 use crate::state::AppState;
 
@@ -176,6 +177,195 @@ pub async fn list_neighborhood_scores(
     Ok(Json(scores))
 }
 
+pub async fn list_data_sources(State(state): State<AppState>) -> ApiResult<Json<Vec<DataSource>>> {
+    let sources = sqlx::query_as::<_, DataSource>(
+        r#"
+        SELECT
+            source_id,
+            name,
+            source_type,
+            url,
+            cadence,
+            reliability,
+            notes,
+            created_at::text AS created_at
+        FROM audit.data_sources
+        ORDER BY name ASC
+        "#,
+    )
+    .fetch_all(&state.pool)
+    .await?;
+
+    Ok(Json(sources))
+}
+
+pub async fn create_data_source(
+    State(state): State<AppState>,
+    Json(payload): Json<CreateDataSource>,
+) -> ApiResult<Json<DataSource>> {
+    payload
+        .validate()
+        .map_err(|message| ApiError::BadRequest(message.to_string()))?;
+
+    let source = sqlx::query_as::<_, DataSource>(
+        r#"
+        INSERT INTO audit.data_sources (
+            name,
+            source_type,
+            url,
+            cadence,
+            reliability,
+            notes
+        )
+        VALUES ($1, $2, $3, $4, COALESCE($5, 'unknown'), COALESCE($6, ''))
+        ON CONFLICT (name) DO UPDATE
+        SET source_type = EXCLUDED.source_type,
+            url = EXCLUDED.url,
+            cadence = EXCLUDED.cadence,
+            reliability = EXCLUDED.reliability,
+            notes = EXCLUDED.notes
+        RETURNING
+            source_id,
+            name,
+            source_type,
+            url,
+            cadence,
+            reliability,
+            notes,
+            created_at::text AS created_at
+        "#,
+    )
+    .bind(payload.name.trim().to_string())
+    .bind(payload.source_type.trim().to_string())
+    .bind(payload.url)
+    .bind(payload.cadence)
+    .bind(payload.reliability)
+    .bind(payload.notes)
+    .fetch_one(&state.pool)
+    .await?;
+
+    Ok(Json(source))
+}
+
+pub async fn list_ingestion_runs(
+    State(state): State<AppState>,
+    Query(query): Query<IngestionRunQuery>,
+) -> ApiResult<Json<Vec<IngestionRun>>> {
+    let runs = sqlx::query_as::<_, IngestionRun>(
+        r#"
+        SELECT
+            r.run_id,
+            r.source_id,
+            s.name AS source_name,
+            r.status,
+            r.started_at::text AS started_at,
+            r.finished_at::text AS finished_at,
+            r.raw_uri,
+            r.row_count,
+            r.error_message
+        FROM audit.ingestion_runs r
+        LEFT JOIN audit.data_sources s ON s.source_id = r.source_id
+        WHERE ($1::bigint IS NULL OR r.source_id = $1)
+        ORDER BY r.started_at DESC, r.run_id DESC
+        LIMIT 100
+        "#,
+    )
+    .bind(query.source_id)
+    .fetch_all(&state.pool)
+    .await?;
+
+    Ok(Json(runs))
+}
+
+pub async fn create_ingestion_run(
+    State(state): State<AppState>,
+    Json(payload): Json<CreateIngestionRun>,
+) -> ApiResult<Json<IngestionRun>> {
+    payload
+        .validate()
+        .map_err(|message| ApiError::BadRequest(message.to_string()))?;
+
+    let run = sqlx::query_as::<_, IngestionRun>(
+        r#"
+        INSERT INTO audit.ingestion_runs (
+            source_id,
+            status,
+            raw_uri,
+            row_count,
+            error_message
+        )
+        VALUES ($1, COALESCE($2, 'running'), $3, $4, $5)
+        RETURNING
+            run_id,
+            source_id,
+            (
+                SELECT name
+                FROM audit.data_sources
+                WHERE source_id = audit.ingestion_runs.source_id
+            ) AS source_name,
+            status,
+            started_at::text AS started_at,
+            finished_at::text AS finished_at,
+            raw_uri,
+            row_count,
+            error_message
+        "#,
+    )
+    .bind(payload.source_id)
+    .bind(payload.status)
+    .bind(payload.raw_uri)
+    .bind(payload.row_count)
+    .bind(payload.error_message)
+    .fetch_one(&state.pool)
+    .await?;
+
+    Ok(Json(run))
+}
+
+pub async fn finish_ingestion_run(
+    State(state): State<AppState>,
+    Path(run_id): Path<i64>,
+    Json(payload): Json<FinishIngestionRun>,
+) -> ApiResult<Json<IngestionRun>> {
+    payload
+        .validate()
+        .map_err(|message| ApiError::BadRequest(message.to_string()))?;
+
+    let run = sqlx::query_as::<_, IngestionRun>(
+        r#"
+        UPDATE audit.ingestion_runs
+        SET status = $2,
+            finished_at = now(),
+            row_count = COALESCE($3, row_count),
+            error_message = $4
+        WHERE run_id = $1
+        RETURNING
+            run_id,
+            source_id,
+            (
+                SELECT name
+                FROM audit.data_sources
+                WHERE source_id = audit.ingestion_runs.source_id
+            ) AS source_name,
+            status,
+            started_at::text AS started_at,
+            finished_at::text AS finished_at,
+            raw_uri,
+            row_count,
+            error_message
+        "#,
+    )
+    .bind(run_id)
+    .bind(payload.status)
+    .bind(payload.row_count)
+    .bind(payload.error_message)
+    .fetch_optional(&state.pool)
+    .await?
+    .ok_or_else(|| ApiError::BadRequest("ingestion run not found".to_string()))?;
+
+    Ok(Json(run))
+}
+
 pub async fn list_watchlist_items(
     State(state): State<AppState>,
 ) -> ApiResult<Json<Vec<WatchlistItem>>> {
@@ -367,7 +557,10 @@ fn validate_month(month: &str) -> ApiResult<()> {
 
 #[cfg(test)]
 mod tests {
-    use crate::models::{CreateWatchlistItem, UpdateWatchlistItem};
+    use crate::models::{
+        CreateDataSource, CreateIngestionRun, CreateWatchlistItem, FinishIngestionRun,
+        UpdateWatchlistItem,
+    };
 
     use super::validate_month;
 
@@ -430,4 +623,79 @@ mod tests {
         .validate()
         .is_ok());
     }
+
+    #[test]
+    fn validates_data_source_payload() {
+        assert!(CreateDataSource {
+            name: "".to_string(),
+            source_type: "official".to_string(),
+            url: None,
+            cadence: None,
+            reliability: None,
+            notes: None,
+        }
+        .validate()
+        .is_err());
+
+        assert!(CreateDataSource {
+            name: "上海市统计局".to_string(),
+            source_type: "official".to_string(),
+            url: Some("https://tjj.sh.gov.cn/".to_string()),
+            cadence: Some("monthly".to_string()),
+            reliability: Some("high".to_string()),
+            notes: None,
+        }
+        .validate()
+        .is_ok());
+
+        assert!(CreateDataSource {
+            name: "unsafe".to_string(),
+            source_type: "manual".to_string(),
+            url: None,
+            cadence: None,
+            reliability: None,
+            notes: Some("contains password marker".to_string()),
+        }
+        .validate()
+        .is_err());
+    }
+
+    #[test]
+    fn validates_ingestion_run_payloads() {
+        assert!(CreateIngestionRun {
+            source_id: Some(1),
+            status: Some("running".to_string()),
+            raw_uri: Some("raw/official/2026-05.csv".to_string()),
+            row_count: Some(10),
+            error_message: None,
+        }
+        .validate()
+        .is_ok());
+
+        assert!(CreateIngestionRun {
+            source_id: Some(1),
+            status: Some("unknown".to_string()),
+            raw_uri: None,
+            row_count: None,
+            error_message: None,
+        }
+        .validate()
+        .is_err());
+
+        assert!(FinishIngestionRun {
+            status: "succeeded".to_string(),
+            row_count: Some(10),
+            error_message: None,
+        }
+        .validate()
+        .is_ok());
+
+        assert!(FinishIngestionRun {
+            status: "running".to_string(),
+            row_count: None,
+            error_message: None,
+        }
+        .validate()
+        .is_err());
+    }
 }