feat: add import templates

src/shanghai_housing/cli.py

@@ -4,8 +4,9 @@ import argparse
 from pathlib import Path
 from typing import Optional
 
-from .db import DEFAULT_DB_PATH, connect, init_db, load_sample_data
+from .db import DEFAULT_DB_PATH, connect, import_template_file, init_db, load_sample_data
 from .indicators import compute_area_scores
+from .import_templates import ImportTemplateError, validate_tabular_file
 from .reporting import format_scores_table, write_monthly_report
 
 
@@ -36,6 +37,38 @@ def build_parser() -> argparse.ArgumentParser:
         help="Output Markdown path. Default: reports/monthly-YYYY-MM.md",
     )
 
+    validate_import_parser = subparsers.add_parser(
+        "validate-import",
+        help="Validate a CSV or Excel import file against a named template.",
+    )
+    validate_import_parser.add_argument(
+        "--template",
+        required=True,
+        help="Template name, for example area_monthly_metrics.",
+    )
+    validate_import_parser.add_argument(
+        "--file",
+        type=Path,
+        required=True,
+        help="CSV, XLSX, or XLSM file to validate.",
+    )
+
+    import_parser = subparsers.add_parser(
+        "import-file",
+        help="Validate and import a CSV or Excel file into the local research database.",
+    )
+    import_parser.add_argument(
+        "--template",
+        required=True,
+        help="Template name, for example area_monthly_metrics.",
+    )
+    import_parser.add_argument(
+        "--file",
+        type=Path,
+        required=True,
+        help="CSV, XLSX, or XLSM file to import.",
+    )
+
     return parser
 
 
@@ -66,5 +99,38 @@ def main(argv: Optional[list[str]] = None) -> int:
         print(f"Generated report: {path}")
         return 0
 
+    if args.command == "validate-import":
+        try:
+            result = validate_tabular_file(args.template, args.file)
+        except ImportTemplateError as exc:
+            print(f"Import validation setup error: {exc}")
+            return 2
+
+        if result.valid:
+            print(
+                f"Import file is valid: template={result.template_name}, "
+                f"rows={result.row_count}"
+            )
+            return 0
+
+        print(
+            f"Import file is invalid: template={result.template_name}, "
+            f"rows={result.row_count}, errors={len(result.errors)}"
+        )
+        for error in result.errors:
+            print(f"row {error.row}, column {error.column}: {error.message}")
+        return 1
+
+    if args.command == "import-file":
+        try:
+            count = import_template_file(args.db, args.template, args.file)
+        except ImportTemplateError as exc:
+            print(f"Import failed: {exc}")
+            return 1
+        print(
+            f"Imported file: template={args.template}, rows={count}, database={args.db}"
+        )
+        return 0
+
     parser.error(f"Unknown command: {args.command}")
     return 2

src/shanghai_housing/db.py

@@ -5,12 +5,20 @@ import sqlite3
 from pathlib import Path
 from typing import Iterable, Mapping, Union
 
+from .import_templates import ImportTemplateError, load_tabular_rows, validate_import_rows
+
 
 PROJECT_ROOT = Path(__file__).resolve().parents[2]
 DEFAULT_DB_PATH = PROJECT_ROOT / "data" / "shanghai_housing.sqlite"
 SCHEMA_PATH = Path(__file__).with_name("schema.sql")
 SAMPLE_DIR = PROJECT_ROOT / "data" / "sample"
 
+SQLITE_IMPORT_TABLES = {
+    "area_monthly_metrics": "area_monthly_metrics",
+    "neighborhood_monthly_metrics": "neighborhood_monthly_metrics",
+    "policy_events": "policy_events",
+}
+
 
 def connect(db_path: Union[Path, str] = DEFAULT_DB_PATH) -> sqlite3.Connection:
     conn = sqlite3.connect(str(db_path))
@@ -73,6 +81,44 @@ def load_csv(conn: sqlite3.Connection, table: str, path: Path) -> int:
     return len(rows)
 
 
+def import_template_file(
+    db_path: Union[Path, str],
+    template_name: str,
+    file_path: Path,
+) -> int:
+    table = SQLITE_IMPORT_TABLES.get(template_name)
+    if table is None:
+        supported = ", ".join(sorted(SQLITE_IMPORT_TABLES))
+        raise ImportTemplateError(
+            f"template '{template_name}' cannot be imported into SQLite; "
+            f"supported templates: {supported}"
+        )
+
+    rows = load_tabular_rows(file_path)
+    result = validate_import_rows(template_name, rows)
+    if not result.valid:
+        first_error = result.errors[0]
+        raise ImportTemplateError(
+            f"validation failed at row {first_error.row}, "
+            f"column {first_error.column}: {first_error.message}"
+        )
+    if not rows:
+        return 0
+
+    init_db(db_path)
+    with connect(db_path) as conn:
+        columns = list(rows[0].keys())
+        placeholders = ", ".join(["?"] * len(columns))
+        column_sql = ", ".join(columns)
+        update_sql = ", ".join([f"{column}=excluded.{column}" for column in columns])
+        sql = (
+            f"INSERT INTO {table} ({column_sql}) VALUES ({placeholders}) "
+            f"ON CONFLICT DO UPDATE SET {update_sql}"
+        )
+        conn.executemany(sql, [tuple(row[column] for column in columns) for row in rows])
+    return len(rows)
+
+
 def fetch_area_metrics(conn: sqlite3.Connection, month: str) -> list[sqlite3.Row]:
     return conn.execute(
         """

src/shanghai_housing/import_templates.py

@@ -0,0 +1,306 @@
+from __future__ import annotations
+
+import csv
+import json
+import re
+from dataclasses import dataclass
+from datetime import date
+from pathlib import Path
+from typing import Any, Mapping
+from urllib.parse import urlparse
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+IMPORT_TEMPLATE_SPEC_PATH = PROJECT_ROOT / "config" / "import_templates.json"
+
+
+@dataclass(frozen=True)
+class ImportValidationError:
+    row: int
+    column: str
+    message: str
+
+
+@dataclass(frozen=True)
+class ImportValidationResult:
+    template_name: str
+    row_count: int
+    valid: bool
+    errors: list[ImportValidationError]
+
+
+class ImportTemplateError(ValueError):
+    pass
+
+
+def load_import_template_specs(
+    path: Path = IMPORT_TEMPLATE_SPEC_PATH,
+) -> dict[str, dict[str, Any]]:
+    with path.open("r", encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def list_template_names(path: Path = IMPORT_TEMPLATE_SPEC_PATH) -> list[str]:
+    return sorted(load_import_template_specs(path).keys())
+
+
+def get_template_spec(
+    template_name: str,
+    path: Path = IMPORT_TEMPLATE_SPEC_PATH,
+) -> dict[str, Any]:
+    specs = load_import_template_specs(path)
+    try:
+        return specs[template_name]
+    except KeyError as exc:
+        valid_names = ", ".join(sorted(specs.keys()))
+        raise ImportTemplateError(
+            f"unknown template '{template_name}', expected one of: {valid_names}"
+        ) from exc
+
+
+def validate_csv_file(
+    template_name: str,
+    file_path: Path,
+    spec_path: Path = IMPORT_TEMPLATE_SPEC_PATH,
+) -> ImportValidationResult:
+    rows = read_csv_rows(file_path)
+    return validate_import_rows(template_name, rows, spec_path=spec_path)
+
+
+def validate_tabular_file(
+    template_name: str,
+    file_path: Path,
+    spec_path: Path = IMPORT_TEMPLATE_SPEC_PATH,
+) -> ImportValidationResult:
+    suffix = file_path.suffix.lower()
+    if suffix == ".csv":
+        rows = read_csv_rows(file_path)
+    elif suffix in {".xlsx", ".xlsm"}:
+        rows = read_excel_rows(file_path)
+    else:
+        raise ImportTemplateError("file must be a .csv, .xlsx, or .xlsm document")
+    return validate_import_rows(template_name, rows, spec_path=spec_path)
+
+
+def load_tabular_rows(file_path: Path) -> list[dict[str, str]]:
+    suffix = file_path.suffix.lower()
+    if suffix == ".csv":
+        return read_csv_rows(file_path)
+    if suffix in {".xlsx", ".xlsm"}:
+        return read_excel_rows(file_path)
+    raise ImportTemplateError("file must be a .csv, .xlsx, or .xlsm document")
+
+
+def read_csv_rows(file_path: Path) -> list[dict[str, str]]:
+    with file_path.open("r", encoding="utf-8-sig", newline="") as handle:
+        reader = csv.DictReader(handle)
+        return [dict(row) for row in reader]
+
+
+def read_excel_rows(file_path: Path) -> list[dict[str, str]]:
+    try:
+        from openpyxl import load_workbook
+    except ImportError as exc:
+        raise ImportTemplateError(
+            "Excel validation requires openpyxl; install project dependencies first"
+        ) from exc
+
+    workbook = load_workbook(file_path, read_only=True, data_only=True)
+    worksheet = workbook.active
+    rows = worksheet.iter_rows(values_only=True)
+    try:
+        headers = [str(value).strip() if value is not None else "" for value in next(rows)]
+    except StopIteration:
+        return []
+
+    records: list[dict[str, str]] = []
+    for values in rows:
+        if values is None or all(value is None or str(value).strip() == "" for value in values):
+            continue
+        record = {
+            header: "" if value is None else str(value).strip()
+            for header, value in zip(headers, values)
+            if header
+        }
+        records.append(record)
+    return records
+
+
+def validate_import_rows(
+    template_name: str,
+    rows: list[Mapping[str, Any]],
+    spec_path: Path = IMPORT_TEMPLATE_SPEC_PATH,
+) -> ImportValidationResult:
+    spec = get_template_spec(template_name, spec_path)
+    columns = spec.get("columns", [])
+    expected_columns = [column["name"] for column in columns]
+    required_columns = [
+        column["name"] for column in columns if bool(column.get("required", False))
+    ]
+    errors: list[ImportValidationError] = []
+
+    seen_columns: set[str] = set()
+    if rows:
+        seen_columns = set(rows[0].keys())
+    missing_headers = [column for column in expected_columns if column not in seen_columns]
+    unexpected_headers = sorted(seen_columns - set(expected_columns))
+
+    for column in missing_headers:
+        errors.append(ImportValidationError(0, column, "missing required header"))
+    for column in unexpected_headers:
+        errors.append(ImportValidationError(0, column, "unexpected header"))
+
+    if not rows:
+        errors.append(ImportValidationError(0, "*", "file has no data rows"))
+
+    for index, row in enumerate(rows, start=2):
+        for column in columns:
+            name = column["name"]
+            value = normalize_value(row.get(name))
+            if name in required_columns and value == "":
+                errors.append(ImportValidationError(index, name, "value is required"))
+                continue
+            if value == "":
+                continue
+            errors.extend(validate_value(index, name, value, column))
+
+    return ImportValidationResult(
+        template_name=template_name,
+        row_count=len(rows),
+        valid=not errors,
+        errors=errors,
+    )
+
+
+def normalize_value(value: Any) -> str:
+    if value is None:
+        return ""
+    return str(value).strip()
+
+
+def validate_value(
+    row_index: int,
+    column_name: str,
+    value: str,
+    column: Mapping[str, Any],
+) -> list[ImportValidationError]:
+    errors: list[ImportValidationError] = []
+    if has_sensitive_text(value):
+        return [
+            ImportValidationError(
+                row_index,
+                column_name,
+                "value appears to contain credentials or a sensitive local path",
+            )
+        ]
+
+    column_type = column["type"]
+    if column_type == "text":
+        return errors
+    if column_type == "month":
+        if not is_valid_month(value):
+            errors.append(ImportValidationError(row_index, column_name, "must be YYYY-MM"))
+        return errors
+    if column_type == "date":
+        if not is_valid_date(value):
+            errors.append(ImportValidationError(row_index, column_name, "must be YYYY-MM-DD"))
+        return errors
+    if column_type == "url":
+        if not is_valid_url(value):
+            errors.append(ImportValidationError(row_index, column_name, "must be http(s) URL"))
+        return errors
+    if column_type == "enum":
+        allowed_values = column.get("allowed_values", [])
+        if value not in allowed_values:
+            allowed = ", ".join(allowed_values)
+            errors.append(
+                ImportValidationError(row_index, column_name, f"must be one of: {allowed}")
+            )
+        return errors
+    if column_type == "integer":
+        parsed = parse_integer(value)
+        if parsed is None:
+            errors.append(ImportValidationError(row_index, column_name, "must be an integer"))
+            return errors
+        errors.extend(validate_numeric_range(row_index, column_name, parsed, column))
+        return errors
+    if column_type == "decimal":
+        parsed = parse_decimal(value)
+        if parsed is None:
+            errors.append(ImportValidationError(row_index, column_name, "must be a number"))
+            return errors
+        errors.extend(validate_numeric_range(row_index, column_name, parsed, column))
+        return errors
+
+    errors.append(
+        ImportValidationError(row_index, column_name, f"unknown type '{column_type}'")
+    )
+    return errors
+
+
+def validate_numeric_range(
+    row_index: int,
+    column_name: str,
+    value: float,
+    column: Mapping[str, Any],
+) -> list[ImportValidationError]:
+    errors: list[ImportValidationError] = []
+    minimum = column.get("min")
+    maximum = column.get("max")
+    if minimum is not None and value < float(minimum):
+        errors.append(
+            ImportValidationError(row_index, column_name, f"must be >= {minimum}")
+        )
+    if maximum is not None and value > float(maximum):
+        errors.append(
+            ImportValidationError(row_index, column_name, f"must be <= {maximum}")
+        )
+    return errors
+
+
+def parse_integer(value: str) -> int | None:
+    if not re.fullmatch(r"-?\d+", value):
+        return None
+    return int(value)
+
+
+def parse_decimal(value: str) -> float | None:
+    try:
+        return float(value)
+    except ValueError:
+        return None
+
+
+def is_valid_month(value: str) -> bool:
+    if not re.fullmatch(r"\d{4}-\d{2}", value):
+        return False
+    month = int(value[5:])
+    return 1 <= month <= 12
+
+
+def is_valid_date(value: str) -> bool:
+    if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
+        return False
+    try:
+        date.fromisoformat(value)
+    except ValueError:
+        return False
+    return True
+
+
+def is_valid_url(value: str) -> bool:
+    parsed = urlparse(value)
+    return parsed.scheme in {"http", "https"} and bool(parsed.netloc)
+
+
+def has_sensitive_text(value: str) -> bool:
+    lower = value.lower()
+    return (
+        "password" in lower
+        or "secret" in lower
+        or "postgres://" in lower
+        or "database_url" in lower
+        or "root_" in lower
+        or value.startswith("/Users/")
+        or value.startswith("/private/")
+    )

src/shanghai_housing/schema.sql

@@ -60,7 +60,8 @@ CREATE TABLE IF NOT EXISTS policy_events (
     title TEXT NOT NULL,
     impact_direction INTEGER NOT NULL CHECK (impact_direction BETWEEN -2 AND 2),
     notes TEXT DEFAULT '',
-    source_url TEXT
+    source_url TEXT,
+    source TEXT NOT NULL DEFAULT 'manual'
 );
 
 CREATE INDEX IF NOT EXISTS idx_area_metrics_month ON area_monthly_metrics(month);