feat: add import templates

apps/api/src/handlers.rs

@@ -3,11 +3,12 @@ use axum::Json;
 use serde::Serialize;
 
 use crate::error::{ApiError, ApiResult};
+use crate::import_templates::validate_import_payload;
 use crate::models::{
     AreaScore, AreaScoreSummary, CreateDataSource, CreateIngestionRun, CreateWatchlistItem,
-    DataSource, FinishIngestionRun, IngestionRun, IngestionRunQuery, MarketOverview, MonthQuery,
-    Neighborhood, NeighborhoodQuery, NeighborhoodScore, NeighborhoodScoreQuery,
-    UpdateWatchlistItem, WatchlistItem,
+    DataSource, FinishIngestionRun, ImportValidationRequest, ImportValidationResponse,
+    IngestionRun, IngestionRunQuery, MarketOverview, MonthQuery, Neighborhood, NeighborhoodQuery,
+    NeighborhoodScore, NeighborhoodScoreQuery, UpdateWatchlistItem, WatchlistItem,
 };
 use crate::state::AppState;
 
@@ -32,6 +33,13 @@ pub async fn ready(State(state): State<AppState>) -> ApiResult<Json<HealthRespon
     }))
 }
 
+pub async fn validate_import(
+    Json(payload): Json<ImportValidationRequest>,
+) -> ApiResult<Json<ImportValidationResponse>> {
+    let result = validate_import_payload(&payload).map_err(ApiError::BadRequest)?;
+    Ok(Json(result))
+}
+
 pub async fn list_area_scores(
     State(state): State<AppState>,
     Query(query): Query<MonthQuery>,

apps/api/src/import_templates.rs

@@ -0,0 +1,389 @@
+use std::collections::{HashMap, HashSet};
+
+use serde::Deserialize;
+use serde_json::Value;
+
+use crate::models::{ImportValidationError, ImportValidationRequest, ImportValidationResponse};
+
+const IMPORT_TEMPLATE_SPEC: &str = include_str!("../../../config/import_templates.json");
+
+#[derive(Debug, Deserialize)]
+struct TemplateSpec {
+    columns: Vec<ColumnSpec>,
+}
+
+#[derive(Debug, Deserialize)]
+struct ColumnSpec {
+    name: String,
+    #[serde(rename = "type")]
+    column_type: String,
+    required: bool,
+    min: Option<f64>,
+    max: Option<f64>,
+    allowed_values: Option<Vec<String>>,
+}
+
+pub fn validate_import_payload(
+    payload: &ImportValidationRequest,
+) -> Result<ImportValidationResponse, String> {
+    let specs = load_specs()?;
+    let spec = specs.get(&payload.template_name).ok_or_else(|| {
+        let mut valid_names = specs.keys().cloned().collect::<Vec<_>>();
+        valid_names.sort();
+        format!(
+            "unknown template '{}', expected one of: {}",
+            payload.template_name,
+            valid_names.join(", ")
+        )
+    })?;
+
+    let mut errors = Vec::new();
+    let expected_columns = spec
+        .columns
+        .iter()
+        .map(|column| column.name.as_str())
+        .collect::<HashSet<_>>();
+    let seen_columns = payload
+        .rows
+        .iter()
+        .flat_map(|row| row.keys().map(String::as_str))
+        .collect::<HashSet<_>>();
+
+    for column in &spec.columns {
+        if !seen_columns.contains(column.name.as_str()) {
+            errors.push(ImportValidationError::new(
+                0,
+                &column.name,
+                "missing required header",
+            ));
+        }
+    }
+
+    let mut unexpected_columns = seen_columns
+        .iter()
+        .copied()
+        .filter(|column| !expected_columns.contains(column))
+        .collect::<Vec<_>>();
+    unexpected_columns.sort_unstable();
+    for column in unexpected_columns {
+        errors.push(ImportValidationError::new(0, column, "unexpected header"));
+    }
+
+    if payload.rows.is_empty() {
+        errors.push(ImportValidationError::new(0, "*", "file has no data rows"));
+    }
+
+    for (index, row) in payload.rows.iter().enumerate() {
+        let row_number = index + 2;
+        for column in &spec.columns {
+            let value = normalize_value(row.get(&column.name));
+            if column.required && value.is_empty() {
+                errors.push(ImportValidationError::new(
+                    row_number,
+                    &column.name,
+                    "value is required",
+                ));
+                continue;
+            }
+            if value.is_empty() {
+                continue;
+            }
+            errors.extend(validate_value(row_number, &column.name, &value, column));
+        }
+    }
+
+    Ok(ImportValidationResponse {
+        template_name: payload.template_name.clone(),
+        row_count: payload.rows.len(),
+        valid: errors.is_empty(),
+        errors,
+    })
+}
+
+fn load_specs() -> Result<HashMap<String, TemplateSpec>, String> {
+    serde_json::from_str(IMPORT_TEMPLATE_SPEC)
+        .map_err(|error| format!("failed to parse import template specs: {error}"))
+}
+
+fn normalize_value(value: Option<&Value>) -> String {
+    match value {
+        None | Some(Value::Null) => String::new(),
+        Some(Value::String(value)) => value.trim().to_string(),
+        Some(Value::Number(value)) => value.to_string(),
+        Some(Value::Bool(value)) => value.to_string(),
+        Some(value) => value.to_string(),
+    }
+}
+
+fn validate_value(
+    row_number: usize,
+    column_name: &str,
+    value: &str,
+    column: &ColumnSpec,
+) -> Vec<ImportValidationError> {
+    if has_sensitive_text(value) {
+        return vec![ImportValidationError::new(
+            row_number,
+            column_name,
+            "value appears to contain credentials or a sensitive local path",
+        )];
+    }
+
+    match column.column_type.as_str() {
+        "text" => Vec::new(),
+        "month" => {
+            if is_valid_month(value) {
+                Vec::new()
+            } else {
+                vec![ImportValidationError::new(
+                    row_number,
+                    column_name,
+                    "must be YYYY-MM",
+                )]
+            }
+        }
+        "date" => {
+            if is_valid_date(value) {
+                Vec::new()
+            } else {
+                vec![ImportValidationError::new(
+                    row_number,
+                    column_name,
+                    "must be YYYY-MM-DD",
+                )]
+            }
+        }
+        "url" => {
+            if is_valid_url(value) {
+                Vec::new()
+            } else {
+                vec![ImportValidationError::new(
+                    row_number,
+                    column_name,
+                    "must be http(s) URL",
+                )]
+            }
+        }
+        "enum" => {
+            let allowed_values = column.allowed_values.as_deref().unwrap_or_default();
+            if allowed_values.iter().any(|allowed| allowed == value) {
+                Vec::new()
+            } else {
+                vec![ImportValidationError::new(
+                    row_number,
+                    column_name,
+                    format!("must be one of: {}", allowed_values.join(", ")),
+                )]
+            }
+        }
+        "integer" => match parse_integer(value) {
+            Some(parsed) => validate_numeric_range(row_number, column_name, parsed as f64, column),
+            _ => vec![ImportValidationError::new(
+                row_number,
+                column_name,
+                "must be an integer",
+            )],
+        },
+        "decimal" => match value.parse::<f64>() {
+            Ok(parsed) if parsed.is_finite() => {
+                validate_numeric_range(row_number, column_name, parsed, column)
+            }
+            _ => vec![ImportValidationError::new(
+                row_number,
+                column_name,
+                "must be a number",
+            )],
+        },
+        unknown => vec![ImportValidationError::new(
+            row_number,
+            column_name,
+            format!("unknown type '{unknown}'"),
+        )],
+    }
+}
+
+fn validate_numeric_range(
+    row_number: usize,
+    column_name: &str,
+    value: f64,
+    column: &ColumnSpec,
+) -> Vec<ImportValidationError> {
+    let mut errors = Vec::new();
+    if let Some(minimum) = column.min {
+        if value < minimum {
+            errors.push(ImportValidationError::new(
+                row_number,
+                column_name,
+                format!("must be >= {}", format_number(minimum)),
+            ));
+        }
+    }
+    if let Some(maximum) = column.max {
+        if value > maximum {
+            errors.push(ImportValidationError::new(
+                row_number,
+                column_name,
+                format!("must be <= {}", format_number(maximum)),
+            ));
+        }
+    }
+    errors
+}
+
+fn format_number(value: f64) -> String {
+    if value.fract() == 0.0 {
+        format!("{}", value as i64)
+    } else {
+        value.to_string()
+    }
+}
+
+fn parse_integer(value: &str) -> Option<i64> {
+    let digits = value.strip_prefix('-').unwrap_or(value);
+    if digits.is_empty() || !digits.chars().all(|item| item.is_ascii_digit()) {
+        return None;
+    }
+    value.parse::<i64>().ok()
+}
+
+fn is_valid_month(value: &str) -> bool {
+    if value.len() != 7 || value.as_bytes()[4] != b'-' {
+        return false;
+    }
+    value[..4].chars().all(|item| item.is_ascii_digit())
+        && value[5..].chars().all(|item| item.is_ascii_digit())
+        && matches!(value[5..].parse::<u8>(), Ok(month) if (1..=12).contains(&month))
+}
+
+fn is_valid_date(value: &str) -> bool {
+    if value.len() != 10 || value.as_bytes()[4] != b'-' || value.as_bytes()[7] != b'-' {
+        return false;
+    }
+    if !value
+        .chars()
+        .enumerate()
+        .all(|(index, item)| index == 4 || index == 7 || item.is_ascii_digit())
+    {
+        return false;
+    }
+
+    let year = value[..4].parse::<i32>().unwrap_or_default();
+    let month = value[5..7].parse::<u8>().unwrap_or_default();
+    let day = value[8..].parse::<u8>().unwrap_or_default();
+    if !(1..=12).contains(&month) {
+        return false;
+    }
+    let max_day = match month {
+        1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
+        4 | 6 | 9 | 11 => 30,
+        2 if is_leap_year(year) => 29,
+        2 => 28,
+        _ => 0,
+    };
+    (1..=max_day).contains(&day)
+}
+
+fn is_leap_year(year: i32) -> bool {
+    (year % 4 == 0 && year % 100 != 0) || year % 400 == 0
+}
+
+fn is_valid_url(value: &str) -> bool {
+    (value.starts_with("http://") || value.starts_with("https://"))
+        && value
+            .split_once("://")
+            .is_some_and(|(_, rest)| rest.contains('.') && !rest.starts_with('.'))
+}
+
+fn has_sensitive_text(value: &str) -> bool {
+    let lower = value.to_ascii_lowercase();
+    lower.contains("password")
+        || lower.contains("secret")
+        || lower.contains("postgres://")
+        || lower.contains("database_url")
+        || lower.contains("root_")
+        || value.starts_with("/Users/")
+        || value.starts_with("/private/")
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+
+    use serde_json::json;
+
+    use crate::models::ImportValidationRequest;
+
+    use super::{is_valid_date, validate_import_payload};
+
+    #[test]
+    fn validates_good_import_rows() {
+        let request = ImportValidationRequest {
+            template_name: "area_monthly_metrics".to_string(),
+            rows: vec![HashMap::from([
+                ("area_id".to_string(), json!("qiantan")),
+                ("month".to_string(), json!("2026-05")),
+                ("transaction_count".to_string(), json!("93")),
+                ("transaction_price_psm".to_string(), json!("122500")),
+                ("listing_count".to_string(), json!("380")),
+                ("listing_price_psm".to_string(), json!("126500")),
+                ("median_days_on_market".to_string(), json!("53")),
+                ("rent_price_psm".to_string(), json!("192")),
+                ("new_supply_units".to_string(), json!("60")),
+                ("land_residential_gfa_sqm".to_string(), json!("0")),
+                ("mortgage_rate_pct".to_string(), json!("3.35")),
+                ("policy_signal".to_string(), json!("1")),
+                ("source".to_string(), json!("manual_research")),
+            ])],
+        };
+
+        let result = validate_import_payload(&request).expect("valid spec");
+
+        assert!(result.valid);
+        assert_eq!(result.row_count, 1);
+        assert!(result.errors.is_empty());
+    }
+
+    #[test]
+    fn reports_bad_import_rows() {
+        let request = ImportValidationRequest {
+            template_name: "area_monthly_metrics".to_string(),
+            rows: vec![HashMap::from([
+                ("area_id".to_string(), json!("")),
+                ("month".to_string(), json!("2026-13")),
+                ("transaction_count".to_string(), json!("-1")),
+                ("transaction_price_psm".to_string(), json!("abc")),
+                ("listing_count".to_string(), json!("10")),
+                ("listing_price_psm".to_string(), json!("100")),
+                ("median_days_on_market".to_string(), json!("30")),
+                ("rent_price_psm".to_string(), json!("3")),
+                ("new_supply_units".to_string(), json!("0")),
+                ("land_residential_gfa_sqm".to_string(), json!("0")),
+                ("mortgage_rate_pct".to_string(), json!("3.35")),
+                ("policy_signal".to_string(), json!("3")),
+                ("source".to_string(), json!("manual")),
+            ])],
+        };
+
+        let result = validate_import_payload(&request).expect("valid spec");
+        let messages = result
+            .errors
+            .iter()
+            .map(|error| (error.column.as_str(), error.message.as_str()))
+            .collect::<Vec<_>>();
+
+        assert!(!result.valid);
+        assert!(messages.contains(&("area_id", "value is required")));
+        assert!(messages.contains(&("month", "must be YYYY-MM")));
+        assert!(messages.contains(&("transaction_count", "must be >= 0")));
+        assert!(messages.contains(&("transaction_price_psm", "must be a number")));
+        assert!(messages.contains(&("policy_signal", "must be <= 2")));
+    }
+
+    #[test]
+    fn validates_real_calendar_dates() {
+        assert!(is_valid_date("2026-02-28"));
+        assert!(!is_valid_date("2026-02-29"));
+        assert!(is_valid_date("2024-02-29"));
+        assert!(!is_valid_date("2026-13-01"));
+    }
+}

apps/api/src/main.rs

@@ -2,6 +2,7 @@ mod config;
 mod db;
 mod error;
 mod handlers;
+mod import_templates;
 mod models;
 mod routes;
 mod state;

apps/api/src/models.rs

@@ -1,4 +1,7 @@
+use std::collections::HashMap;
+
 use serde::{Deserialize, Serialize};
+use serde_json::Value;
 use sqlx::FromRow;
 
 #[derive(Debug, Deserialize)]
@@ -22,6 +25,37 @@ pub struct IngestionRunQuery {
     pub source_id: Option<i64>,
 }
 
+#[derive(Debug, Deserialize)]
+pub struct ImportValidationRequest {
+    pub template_name: String,
+    pub rows: Vec<HashMap<String, Value>>,
+}
+
+#[derive(Debug, Clone, Serialize, PartialEq)]
+pub struct ImportValidationError {
+    pub row: usize,
+    pub column: String,
+    pub message: String,
+}
+
+impl ImportValidationError {
+    pub fn new(row: usize, column: impl Into<String>, message: impl Into<String>) -> Self {
+        Self {
+            row,
+            column: column.into(),
+            message: message.into(),
+        }
+    }
+}
+
+#[derive(Debug, Serialize)]
+pub struct ImportValidationResponse {
+    pub template_name: String,
+    pub row_count: usize,
+    pub valid: bool,
+    pub errors: Vec<ImportValidationError>,
+}
+
 #[derive(Debug, Clone, Serialize, FromRow)]
 pub struct AreaScore {
     pub area_id: String,

apps/api/src/routes.rs

@@ -7,7 +7,7 @@ use crate::handlers::{
     archive_watchlist_item, create_data_source, create_ingestion_run, create_watchlist_item,
     finish_ingestion_run, get_neighborhood, health, list_area_scores, list_data_sources,
     list_ingestion_runs, list_neighborhood_scores, list_neighborhoods, list_watchlist_items,
-    market_overview, ready, update_watchlist_item,
+    market_overview, ready, update_watchlist_item, validate_import,
 };
 use crate::state::AppState;
 
@@ -31,6 +31,7 @@ pub fn build_router(state: AppState) -> Router {
                     "/ingestion-runs/{run_id}/finish",
                     patch(finish_ingestion_run),
                 )
+                .route("/imports/validate", axum::routing::post(validate_import))
                 .route("/market/overview", get(market_overview))
                 .route("/neighborhoods", get(list_neighborhoods))
                 .route("/neighborhoods/scores", get(list_neighborhood_scores))