feat: add raw artifact audit trail

This commit is contained in:
2026-06-24 09:38:15 +08:00
parent 6a17c9a336
commit a770070b34
12 changed files with 670 additions and 16 deletions

View File

@@ -60,6 +60,9 @@ POST /api/v1/data-sources
GET /api/v1/ingestion-runs GET /api/v1/ingestion-runs
POST /api/v1/ingestion-runs POST /api/v1/ingestion-runs
PATCH /api/v1/ingestion-runs/{run_id}/finish PATCH /api/v1/ingestion-runs/{run_id}/finish
GET /api/v1/raw-artifacts
POST /api/v1/raw-artifacts
POST /api/v1/imports/validate
GET /api/v1/areas/scores?month=2026-05 GET /api/v1/areas/scores?month=2026-05
GET /api/v1/market/overview?month=2026-05 GET /api/v1/market/overview?month=2026-05
GET /api/v1/neighborhoods GET /api/v1/neighborhoods

View File

@@ -0,0 +1,24 @@
CREATE TABLE IF NOT EXISTS audit.raw_artifacts (
artifact_id BIGSERIAL PRIMARY KEY,
run_id BIGINT REFERENCES audit.ingestion_runs(run_id),
source_id BIGINT REFERENCES audit.data_sources(source_id),
original_filename TEXT NOT NULL,
raw_uri TEXT NOT NULL,
sha256 TEXT NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'),
size_bytes BIGINT NOT NULL CHECK (size_bytes >= 0),
mime_type TEXT,
uploaded_by TEXT NOT NULL DEFAULT 'system',
notes TEXT NOT NULL DEFAULT '',
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
UNIQUE (sha256)
);
ALTER TABLE audit.ingestion_runs
ADD COLUMN IF NOT EXISTS raw_artifact_id BIGINT REFERENCES audit.raw_artifacts(artifact_id);
CREATE INDEX IF NOT EXISTS idx_raw_artifacts_run_id
ON audit.raw_artifacts(run_id);
CREATE INDEX IF NOT EXISTS idx_raw_artifacts_source_id
ON audit.raw_artifacts(source_id);
CREATE INDEX IF NOT EXISTS idx_raw_artifacts_created_at
ON audit.raw_artifacts(created_at DESC);

View File

@@ -5,10 +5,11 @@ use serde::Serialize;
use crate::error::{ApiError, ApiResult}; use crate::error::{ApiError, ApiResult};
use crate::import_templates::validate_import_payload; use crate::import_templates::validate_import_payload;
use crate::models::{ use crate::models::{
AreaScore, AreaScoreSummary, CreateDataSource, CreateIngestionRun, CreateWatchlistItem, AreaScore, AreaScoreSummary, CreateDataSource, CreateIngestionRun, CreateRawArtifact,
DataSource, FinishIngestionRun, ImportValidationRequest, ImportValidationResponse, CreateWatchlistItem, DataSource, FinishIngestionRun, ImportValidationRequest,
IngestionRun, IngestionRunQuery, MarketOverview, MonthQuery, Neighborhood, NeighborhoodQuery, ImportValidationResponse, IngestionRun, IngestionRunQuery, MarketOverview, MonthQuery,
NeighborhoodScore, NeighborhoodScoreQuery, UpdateWatchlistItem, WatchlistItem, Neighborhood, NeighborhoodQuery, NeighborhoodScore, NeighborhoodScoreQuery, RawArtifact,
RawArtifactQuery, UpdateWatchlistItem, WatchlistItem,
}; };
use crate::state::AppState; use crate::state::AppState;
@@ -265,6 +266,7 @@ pub async fn list_ingestion_runs(
r.run_id, r.run_id,
r.source_id, r.source_id,
s.name AS source_name, s.name AS source_name,
r.raw_artifact_id,
r.status, r.status,
r.started_at::text AS started_at, r.started_at::text AS started_at,
r.finished_at::text AS finished_at, r.finished_at::text AS finished_at,
@@ -299,10 +301,11 @@ pub async fn create_ingestion_run(
source_id, source_id,
status, status,
raw_uri, raw_uri,
raw_artifact_id,
row_count, row_count,
error_message error_message
) )
VALUES ($1, COALESCE($2, 'running'), $3, $4, $5) VALUES ($1, COALESCE($2, 'running'), $3, $4, $5, $6)
RETURNING RETURNING
run_id, run_id,
source_id, source_id,
@@ -311,6 +314,7 @@ pub async fn create_ingestion_run(
FROM audit.data_sources FROM audit.data_sources
WHERE source_id = audit.ingestion_runs.source_id WHERE source_id = audit.ingestion_runs.source_id
) AS source_name, ) AS source_name,
raw_artifact_id,
status, status,
started_at::text AS started_at, started_at::text AS started_at,
finished_at::text AS finished_at, finished_at::text AS finished_at,
@@ -322,6 +326,7 @@ pub async fn create_ingestion_run(
.bind(payload.source_id) .bind(payload.source_id)
.bind(payload.status) .bind(payload.status)
.bind(payload.raw_uri) .bind(payload.raw_uri)
.bind(payload.raw_artifact_id)
.bind(payload.row_count) .bind(payload.row_count)
.bind(payload.error_message) .bind(payload.error_message)
.fetch_one(&state.pool) .fetch_one(&state.pool)
@@ -355,6 +360,7 @@ pub async fn finish_ingestion_run(
FROM audit.data_sources FROM audit.data_sources
WHERE source_id = audit.ingestion_runs.source_id WHERE source_id = audit.ingestion_runs.source_id
) AS source_name, ) AS source_name,
raw_artifact_id,
status, status,
started_at::text AS started_at, started_at::text AS started_at,
finished_at::text AS finished_at, finished_at::text AS finished_at,
@@ -374,6 +380,154 @@ pub async fn finish_ingestion_run(
Ok(Json(run)) Ok(Json(run))
} }
pub async fn list_raw_artifacts(
State(state): State<AppState>,
Query(query): Query<RawArtifactQuery>,
) -> ApiResult<Json<Vec<RawArtifact>>> {
if let Some(sha256) = query.sha256.as_deref() {
validate_sha256(sha256)?;
}
let artifacts = sqlx::query_as::<_, RawArtifact>(
r#"
SELECT
a.artifact_id,
a.run_id,
a.source_id,
s.name AS source_name,
a.original_filename,
a.raw_uri,
a.sha256,
a.size_bytes,
a.mime_type,
a.uploaded_by,
a.notes,
a.created_at::text AS created_at
FROM audit.raw_artifacts a
LEFT JOIN audit.data_sources s ON s.source_id = a.source_id
WHERE ($1::bigint IS NULL OR a.source_id = $1)
AND (
$2::bigint IS NULL
OR a.run_id = $2
OR EXISTS (
SELECT 1
FROM audit.ingestion_runs r
WHERE r.run_id = $2
AND r.raw_artifact_id = a.artifact_id
)
)
AND ($3::text IS NULL OR a.sha256 = $3)
ORDER BY a.created_at DESC, a.artifact_id DESC
LIMIT 100
"#,
)
.bind(query.source_id)
.bind(query.run_id)
.bind(query.sha256)
.fetch_all(&state.pool)
.await?;
Ok(Json(artifacts))
}
pub async fn create_raw_artifact(
State(state): State<AppState>,
Json(payload): Json<CreateRawArtifact>,
) -> ApiResult<Json<RawArtifact>> {
payload
.validate()
.map_err(|message| ApiError::BadRequest(message.to_string()))?;
let mut tx = state.pool.begin().await?;
let artifact = sqlx::query_as::<_, RawArtifact>(
r#"
INSERT INTO audit.raw_artifacts (
run_id,
source_id,
original_filename,
raw_uri,
sha256,
size_bytes,
mime_type,
uploaded_by,
notes
)
VALUES (
$1,
COALESCE(
$2,
(
SELECT source_id
FROM audit.ingestion_runs
WHERE run_id = $1
)
),
$3,
$4,
$5,
$6,
$7,
COALESCE($8, 'system'),
COALESCE($9, '')
)
ON CONFLICT (sha256) DO UPDATE
SET run_id = COALESCE(audit.raw_artifacts.run_id, EXCLUDED.run_id),
source_id = COALESCE(audit.raw_artifacts.source_id, EXCLUDED.source_id),
notes = CASE
WHEN audit.raw_artifacts.notes = '' THEN EXCLUDED.notes
ELSE audit.raw_artifacts.notes
END
RETURNING
artifact_id,
run_id,
source_id,
(
SELECT name
FROM audit.data_sources
WHERE source_id = audit.raw_artifacts.source_id
) AS source_name,
original_filename,
raw_uri,
sha256,
size_bytes,
mime_type,
uploaded_by,
notes,
created_at::text AS created_at
"#,
)
.bind(payload.run_id)
.bind(payload.source_id)
.bind(payload.original_filename.trim().to_string())
.bind(payload.raw_uri.trim().to_string())
.bind(payload.sha256.trim().to_string())
.bind(payload.size_bytes)
.bind(payload.mime_type)
.bind(payload.uploaded_by)
.bind(payload.notes)
.fetch_one(&mut *tx)
.await?;
if let Some(run_id) = payload.run_id {
sqlx::query(
r#"
UPDATE audit.ingestion_runs
SET raw_artifact_id = $2,
raw_uri = COALESCE(raw_uri, $3)
WHERE run_id = $1
"#,
)
.bind(run_id)
.bind(artifact.artifact_id)
.bind(&artifact.raw_uri)
.execute(&mut *tx)
.await?;
}
tx.commit().await?;
Ok(Json(artifact))
}
pub async fn list_watchlist_items( pub async fn list_watchlist_items(
State(state): State<AppState>, State(state): State<AppState>,
) -> ApiResult<Json<Vec<WatchlistItem>>> { ) -> ApiResult<Json<Vec<WatchlistItem>>> {
@@ -563,14 +717,29 @@ fn validate_month(month: &str) -> ApiResult<()> {
} }
} }
fn validate_sha256(value: &str) -> ApiResult<()> {
let valid = value.len() == 64
&& value
.chars()
.all(|item| item.is_ascii_hexdigit() && !item.is_ascii_uppercase());
if valid {
Ok(())
} else {
Err(ApiError::BadRequest(
"sha256 must be a lowercase 64-character hex digest".to_string(),
))
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::models::{ use crate::models::{
CreateDataSource, CreateIngestionRun, CreateWatchlistItem, FinishIngestionRun, CreateDataSource, CreateIngestionRun, CreateRawArtifact, CreateWatchlistItem,
UpdateWatchlistItem, FinishIngestionRun, UpdateWatchlistItem,
}; };
use super::validate_month; use super::{validate_month, validate_sha256};
#[test] #[test]
fn accepts_valid_month() { fn accepts_valid_month() {
@@ -674,6 +843,7 @@ mod tests {
source_id: Some(1), source_id: Some(1),
status: Some("running".to_string()), status: Some("running".to_string()),
raw_uri: Some("raw/official/2026-05.csv".to_string()), raw_uri: Some("raw/official/2026-05.csv".to_string()),
raw_artifact_id: None,
row_count: Some(10), row_count: Some(10),
error_message: None, error_message: None,
} }
@@ -684,6 +854,7 @@ mod tests {
source_id: Some(1), source_id: Some(1),
status: Some("unknown".to_string()), status: Some("unknown".to_string()),
raw_uri: None, raw_uri: None,
raw_artifact_id: None,
row_count: None, row_count: None,
error_message: None, error_message: None,
} }
@@ -706,4 +877,42 @@ mod tests {
.validate() .validate()
.is_err()); .is_err());
} }
#[test]
fn validates_raw_artifact_payload() {
assert!(CreateRawArtifact {
run_id: None,
source_id: None,
original_filename: "area.csv".to_string(),
raw_uri: "raw://sha256/area.csv".to_string(),
sha256: "a".repeat(64),
size_bytes: 128,
mime_type: Some("text/csv".to_string()),
uploaded_by: Some("researcher".to_string()),
notes: None,
}
.validate()
.is_ok());
assert!(CreateRawArtifact {
run_id: None,
source_id: None,
original_filename: "area.csv".to_string(),
raw_uri: "raw://sha256/area.csv".to_string(),
sha256: "A".repeat(64),
size_bytes: 128,
mime_type: None,
uploaded_by: None,
notes: None,
}
.validate()
.is_err());
}
#[test]
fn validates_sha256_query_value() {
assert!(validate_sha256(&"0".repeat(64)).is_ok());
assert!(validate_sha256(&"G".repeat(64)).is_err());
assert!(validate_sha256("short").is_err());
}
} }

View File

@@ -25,6 +25,13 @@ pub struct IngestionRunQuery {
pub source_id: Option<i64>, pub source_id: Option<i64>,
} }
#[derive(Debug, Deserialize)]
pub struct RawArtifactQuery {
pub source_id: Option<i64>,
pub run_id: Option<i64>,
pub sha256: Option<String>,
}
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
pub struct ImportValidationRequest { pub struct ImportValidationRequest {
pub template_name: String, pub template_name: String,
@@ -211,6 +218,7 @@ pub struct IngestionRun {
pub run_id: i64, pub run_id: i64,
pub source_id: Option<i64>, pub source_id: Option<i64>,
pub source_name: Option<String>, pub source_name: Option<String>,
pub raw_artifact_id: Option<i64>,
pub status: String, pub status: String,
pub started_at: String, pub started_at: String,
pub finished_at: Option<String>, pub finished_at: Option<String>,
@@ -224,6 +232,7 @@ pub struct CreateIngestionRun {
pub source_id: Option<i64>, pub source_id: Option<i64>,
pub status: Option<String>, pub status: Option<String>,
pub raw_uri: Option<String>, pub raw_uri: Option<String>,
pub raw_artifact_id: Option<i64>,
pub row_count: Option<i32>, pub row_count: Option<i32>,
pub error_message: Option<String>, pub error_message: Option<String>,
} }
@@ -250,6 +259,67 @@ impl CreateIngestionRun {
} }
} }
#[derive(Debug, Clone, Serialize, FromRow)]
pub struct RawArtifact {
pub artifact_id: i64,
pub run_id: Option<i64>,
pub source_id: Option<i64>,
pub source_name: Option<String>,
pub original_filename: String,
pub raw_uri: String,
pub sha256: String,
pub size_bytes: i64,
pub mime_type: Option<String>,
pub uploaded_by: String,
pub notes: String,
pub created_at: String,
}
#[derive(Debug, Deserialize)]
pub struct CreateRawArtifact {
pub run_id: Option<i64>,
pub source_id: Option<i64>,
pub original_filename: String,
pub raw_uri: String,
pub sha256: String,
pub size_bytes: i64,
pub mime_type: Option<String>,
pub uploaded_by: Option<String>,
pub notes: Option<String>,
}
impl CreateRawArtifact {
pub fn validate(&self) -> Result<(), &'static str> {
if self.original_filename.trim().is_empty() {
return Err("original_filename is required");
}
if self.raw_uri.trim().is_empty() {
return Err("raw_uri is required");
}
if !is_valid_sha256(&self.sha256) {
return Err("sha256 must be a lowercase 64-character hex digest");
}
if self.size_bytes < 0 {
return Err("size_bytes must be non-negative");
}
for value in [
Some(self.original_filename.as_str()),
Some(self.raw_uri.as_str()),
self.mime_type.as_deref(),
self.uploaded_by.as_deref(),
self.notes.as_deref(),
]
.into_iter()
.flatten()
{
if has_sensitive_text(value) {
return Err("payload contains sensitive text");
}
}
Ok(())
}
}
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
pub struct FinishIngestionRun { pub struct FinishIngestionRun {
pub status: String, pub status: String,
@@ -319,6 +389,13 @@ fn has_sensitive_text(value: &str) -> bool {
|| value.starts_with("/private/") || value.starts_with("/private/")
} }
fn is_valid_sha256(value: &str) -> bool {
value.len() == 64
&& value
.chars()
.all(|item| item.is_ascii_hexdigit() && !item.is_ascii_uppercase())
}
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
pub struct UpdateWatchlistItem { pub struct UpdateWatchlistItem {
pub target_price_psm: Option<f64>, pub target_price_psm: Option<f64>,

View File

@@ -4,10 +4,11 @@ use tower_http::cors::CorsLayer;
use tower_http::trace::TraceLayer; use tower_http::trace::TraceLayer;
use crate::handlers::{ use crate::handlers::{
archive_watchlist_item, create_data_source, create_ingestion_run, create_watchlist_item, archive_watchlist_item, create_data_source, create_ingestion_run, create_raw_artifact,
finish_ingestion_run, get_neighborhood, health, list_area_scores, list_data_sources, create_watchlist_item, finish_ingestion_run, get_neighborhood, health, list_area_scores,
list_ingestion_runs, list_neighborhood_scores, list_neighborhoods, list_watchlist_items, list_data_sources, list_ingestion_runs, list_neighborhood_scores, list_neighborhoods,
market_overview, ready, update_watchlist_item, validate_import, list_raw_artifacts, list_watchlist_items, market_overview, ready, update_watchlist_item,
validate_import,
}; };
use crate::state::AppState; use crate::state::AppState;
@@ -31,6 +32,10 @@ pub fn build_router(state: AppState) -> Router {
"/ingestion-runs/{run_id}/finish", "/ingestion-runs/{run_id}/finish",
patch(finish_ingestion_run), patch(finish_ingestion_run),
) )
.route(
"/raw-artifacts",
get(list_raw_artifacts).post(create_raw_artifact),
)
.route("/imports/validate", axum::routing::post(validate_import)) .route("/imports/validate", axum::routing::post(validate_import))
.route("/market/overview", get(market_overview)) .route("/market/overview", get(market_overview))
.route("/neighborhoods", get(list_neighborhoods)) .route("/neighborhoods", get(list_neighborhoods))

View File

@@ -27,12 +27,14 @@ import {
archiveWatchlistItem, archiveWatchlistItem,
createDataSource, createDataSource,
createIngestionRun, createIngestionRun,
createRawArtifact,
createWatchlistItem, createWatchlistItem,
fetchDataSources, fetchDataSources,
fetchAreaScores, fetchAreaScores,
fetchIngestionRuns, fetchIngestionRuns,
fetchMarketOverview, fetchMarketOverview,
fetchNeighborhoodScores, fetchNeighborhoodScores,
fetchRawArtifacts,
fetchWatchlistItems, fetchWatchlistItems,
finishIngestionRun, finishIngestionRun,
} from "@/lib/api"; } from "@/lib/api";
@@ -40,10 +42,12 @@ import type {
AreaScore, AreaScore,
CreateDataSource, CreateDataSource,
CreateIngestionRun, CreateIngestionRun,
CreateRawArtifact,
CreateWatchlistItem, CreateWatchlistItem,
DataSource, DataSource,
IngestionRun, IngestionRun,
NeighborhoodScore, NeighborhoodScore,
RawArtifact,
WatchlistItem, WatchlistItem,
} from "@/lib/api"; } from "@/lib/api";
import { formatNumber, formatPrice } from "@/lib/utils"; import { formatNumber, formatPrice } from "@/lib/utils";
@@ -88,6 +92,10 @@ export function MarketDashboard() {
queryKey: ["ingestion-runs"], queryKey: ["ingestion-runs"],
queryFn: fetchIngestionRuns, queryFn: fetchIngestionRuns,
}); });
const rawArtifacts = useQuery({
queryKey: ["raw-artifacts"],
queryFn: fetchRawArtifacts,
});
const createMutation = useMutation({ const createMutation = useMutation({
mutationFn: createWatchlistItem, mutationFn: createWatchlistItem,
onSuccess: async () => { onSuccess: async () => {
@@ -112,6 +120,15 @@ export function MarketDashboard() {
await queryClient.invalidateQueries({ queryKey: ["ingestion-runs"] }); await queryClient.invalidateQueries({ queryKey: ["ingestion-runs"] });
}, },
}); });
const createRawArtifactMutation = useMutation({
mutationFn: createRawArtifact,
onSuccess: async () => {
await Promise.all([
queryClient.invalidateQueries({ queryKey: ["raw-artifacts"] }),
queryClient.invalidateQueries({ queryKey: ["ingestion-runs"] }),
]);
},
});
const finishRunMutation = useMutation({ const finishRunMutation = useMutation({
mutationFn: ({ id, rowCount }: { id: number; rowCount?: number }) => mutationFn: ({ id, rowCount }: { id: number; rowCount?: number }) =>
finishIngestionRun(id, { status: "succeeded", row_count: rowCount }), finishIngestionRun(id, { status: "succeeded", row_count: rowCount }),
@@ -127,13 +144,15 @@ export function MarketDashboard() {
watchlist.isLoading || watchlist.isLoading ||
dataSources.isLoading || dataSources.isLoading ||
ingestionRuns.isLoading; ingestionRuns.isLoading;
const importLoading = ingestionRuns.isLoading || rawArtifacts.isLoading;
const hasError = const hasError =
overview.isError || overview.isError ||
scores.isError || scores.isError ||
neighborhoodScores.isError || neighborhoodScores.isError ||
watchlist.isError || watchlist.isError ||
dataSources.isError || dataSources.isError ||
ingestionRuns.isError; ingestionRuns.isError ||
rawArtifacts.isError;
return ( return (
<main className="min-h-screen px-4 py-5 sm:px-6 lg:px-8"> <main className="min-h-screen px-4 py-5 sm:px-6 lg:px-8">
@@ -156,6 +175,7 @@ export function MarketDashboard() {
void watchlist.refetch(); void watchlist.refetch();
void dataSources.refetch(); void dataSources.refetch();
void ingestionRuns.refetch(); void ingestionRuns.refetch();
void rawArtifacts.refetch();
}} }}
disabled={isLoading} disabled={isLoading}
aria-label="刷新" aria-label="刷新"
@@ -276,12 +296,16 @@ export function MarketDashboard() {
<ImportCenterPanel <ImportCenterPanel
sources={dataSources.data ?? []} sources={dataSources.data ?? []}
runs={ingestionRuns.data ?? []} runs={ingestionRuns.data ?? []}
artifacts={rawArtifacts.data ?? []}
loading={isLoading} loading={isLoading}
importLoading={importLoading}
creatingSource={createDataSourceMutation.isPending} creatingSource={createDataSourceMutation.isPending}
creatingRun={createRunMutation.isPending} creatingRun={createRunMutation.isPending}
creatingArtifact={createRawArtifactMutation.isPending}
finishingRunId={finishRunMutation.variables?.id ?? null} finishingRunId={finishRunMutation.variables?.id ?? null}
onCreateSource={(payload) => createDataSourceMutation.mutate(payload)} onCreateSource={(payload) => createDataSourceMutation.mutate(payload)}
onCreateRun={(payload) => createRunMutation.mutate(payload)} onCreateRun={(payload) => createRunMutation.mutate(payload)}
onCreateArtifact={(payload) => createRawArtifactMutation.mutate(payload)}
onFinishRun={(id, rowCount) => finishRunMutation.mutate({ id, rowCount })} onFinishRun={(id, rowCount) => finishRunMutation.mutate({ id, rowCount })}
/> />
</div> </div>
@@ -292,22 +316,30 @@ export function MarketDashboard() {
function ImportCenterPanel({ function ImportCenterPanel({
sources, sources,
runs, runs,
artifacts,
loading, loading,
importLoading,
creatingSource, creatingSource,
creatingRun, creatingRun,
creatingArtifact,
finishingRunId, finishingRunId,
onCreateSource, onCreateSource,
onCreateRun, onCreateRun,
onCreateArtifact,
onFinishRun, onFinishRun,
}: { }: {
sources: DataSource[]; sources: DataSource[];
runs: IngestionRun[]; runs: IngestionRun[];
artifacts: RawArtifact[];
loading: boolean; loading: boolean;
importLoading: boolean;
creatingSource: boolean; creatingSource: boolean;
creatingRun: boolean; creatingRun: boolean;
creatingArtifact: boolean;
finishingRunId: number | null; finishingRunId: number | null;
onCreateSource: (payload: CreateDataSource) => void; onCreateSource: (payload: CreateDataSource) => void;
onCreateRun: (payload: CreateIngestionRun) => void; onCreateRun: (payload: CreateIngestionRun) => void;
onCreateArtifact: (payload: CreateRawArtifact) => void;
onFinishRun: (id: number, rowCount?: number) => void; onFinishRun: (id: number, rowCount?: number) => void;
}) { }) {
const [sourceName, setSourceName] = useState(""); const [sourceName, setSourceName] = useState("");
@@ -316,6 +348,12 @@ function ImportCenterPanel({
const [sourceId, setSourceId] = useState(""); const [sourceId, setSourceId] = useState("");
const [rawUri, setRawUri] = useState(""); const [rawUri, setRawUri] = useState("");
const [rowCount, setRowCount] = useState(""); const [rowCount, setRowCount] = useState("");
const [artifactRunId, setArtifactRunId] = useState("");
const [artifactSourceId, setArtifactSourceId] = useState("");
const [artifactFilename, setArtifactFilename] = useState("");
const [artifactRawUri, setArtifactRawUri] = useState("");
const [artifactSha256, setArtifactSha256] = useState("");
const [artifactSizeBytes, setArtifactSizeBytes] = useState("");
function submitSource(event: React.FormEvent<HTMLFormElement>) { function submitSource(event: React.FormEvent<HTMLFormElement>) {
event.preventDefault(); event.preventDefault();
@@ -348,6 +386,36 @@ function ImportCenterPanel({
setRawUri(""); setRawUri("");
} }
function submitArtifact(event: React.FormEvent<HTMLFormElement>) {
event.preventDefault();
const parsedRunId = Number(artifactRunId);
const parsedSourceId = Number(artifactSourceId);
const parsedSizeBytes = Number(artifactSizeBytes);
if (
!artifactFilename.trim() ||
!artifactRawUri.trim() ||
artifactSha256.trim().length !== 64 ||
!Number.isFinite(parsedSizeBytes) ||
parsedSizeBytes < 0
) {
return;
}
onCreateArtifact({
run_id: Number.isFinite(parsedRunId) && parsedRunId > 0 ? parsedRunId : undefined,
source_id:
Number.isFinite(parsedSourceId) && parsedSourceId > 0 ? parsedSourceId : undefined,
original_filename: artifactFilename.trim(),
raw_uri: artifactRawUri.trim(),
sha256: artifactSha256.trim(),
size_bytes: parsedSizeBytes,
uploaded_by: "dashboard",
});
setArtifactFilename("");
setArtifactRawUri("");
setArtifactSha256("");
setArtifactSizeBytes("");
}
return ( return (
<Card> <Card>
<CardHeader className="flex flex-col gap-3 xl:flex-row xl:items-start xl:justify-between"> <CardHeader className="flex flex-col gap-3 xl:flex-row xl:items-start xl:justify-between">
@@ -415,7 +483,7 @@ function ImportCenterPanel({
<DataSourceTable sources={sources} loading={loading} /> <DataSourceTable sources={sources} loading={loading} />
<IngestionRunTable <IngestionRunTable
runs={runs} runs={runs}
loading={loading} loading={importLoading}
rowCount={rowCount} rowCount={rowCount}
onRowCountChange={setRowCount} onRowCountChange={setRowCount}
finishingRunId={finishingRunId} finishingRunId={finishingRunId}
@@ -425,6 +493,73 @@ function ImportCenterPanel({
setRowCount(""); setRowCount("");
}} }}
/> />
<div className="xl:col-span-2">
<form className="mb-3 grid gap-2 lg:grid-cols-[120px_120px_1fr_1fr_1.5fr_120px_44px]" onSubmit={submitArtifact}>
<select
className="h-9 rounded-md border border-[var(--border)] bg-white px-3 text-sm"
value={artifactRunId}
onChange={(event) => setArtifactRunId(event.target.value)}
aria-label="原始文件批次"
>
<option value=""></option>
{runs.map((run) => (
<option key={run.run_id} value={run.run_id}>
#{run.run_id}
</option>
))}
</select>
<select
className="h-9 rounded-md border border-[var(--border)] bg-white px-3 text-sm"
value={artifactSourceId}
onChange={(event) => setArtifactSourceId(event.target.value)}
aria-label="原始文件数据源"
>
<option value=""></option>
{sources.map((source) => (
<option key={source.source_id} value={source.source_id}>
{source.name}
</option>
))}
</select>
<input
className="h-9 rounded-md border border-[var(--border)] bg-white px-3 text-sm"
placeholder="文件名"
value={artifactFilename}
onChange={(event) => setArtifactFilename(event.target.value)}
/>
<input
className="h-9 rounded-md border border-[var(--border)] bg-white px-3 text-sm"
placeholder="raw uri"
value={artifactRawUri}
onChange={(event) => setArtifactRawUri(event.target.value)}
/>
<input
className="h-9 rounded-md border border-[var(--border)] bg-white px-3 text-sm"
placeholder="sha256"
value={artifactSha256}
onChange={(event) => setArtifactSha256(event.target.value)}
/>
<input
className="h-9 rounded-md border border-[var(--border)] bg-white px-3 text-sm"
inputMode="numeric"
placeholder="字节"
value={artifactSizeBytes}
onChange={(event) => setArtifactSizeBytes(event.target.value)}
/>
<Button
disabled={
creatingArtifact ||
!artifactFilename.trim() ||
artifactSha256.trim().length !== 64
}
aria-label="登记原始文件"
title="登记原始文件"
>
<Plus className="h-4 w-4" />
</Button>
</form>
<RawArtifactTable artifacts={artifacts} loading={importLoading} />
</div>
</CardContent> </CardContent>
</Card> </Card>
); );
@@ -539,6 +674,47 @@ function IngestionRunTable({
); );
} }
function RawArtifactTable({
artifacts,
loading,
}: {
artifacts: RawArtifact[];
loading: boolean;
}) {
if (loading) {
return <div className="h-48 rounded-md bg-[var(--muted)]" />;
}
return (
<div className="overflow-x-auto rounded-md border border-[var(--border)]">
<Table>
<TableHeader>
<TableRow>
<TableHead></TableHead>
<TableHead></TableHead>
<TableHead></TableHead>
<TableHead>SHA-256</TableHead>
<TableHead className="text-right"></TableHead>
</TableRow>
</TableHeader>
<TableBody>
{artifacts.map((artifact) => (
<TableRow key={artifact.artifact_id}>
<TableCell className="font-medium">{artifact.original_filename}</TableCell>
<TableCell>{artifact.source_name ?? "-"}</TableCell>
<TableCell>{artifact.run_id ? `#${artifact.run_id}` : "-"}</TableCell>
<TableCell className="max-w-80 truncate font-mono text-xs text-[var(--muted-foreground)]">
{artifact.sha256}
</TableCell>
<TableCell className="text-right">{formatNumber(artifact.size_bytes)}</TableCell>
</TableRow>
))}
</TableBody>
</Table>
</div>
);
}
function NeighborhoodRankingPanel({ function NeighborhoodRankingPanel({
scores, scores,
neighborhoodScores, neighborhoodScores,

View File

@@ -109,6 +109,7 @@ export type IngestionRun = {
run_id: number; run_id: number;
source_id: number | null; source_id: number | null;
source_name: string | null; source_name: string | null;
raw_artifact_id: number | null;
status: string; status: string;
started_at: string; started_at: string;
finished_at: string | null; finished_at: string | null;
@@ -121,6 +122,7 @@ export type CreateIngestionRun = {
source_id?: number; source_id?: number;
status?: string; status?: string;
raw_uri?: string; raw_uri?: string;
raw_artifact_id?: number;
row_count?: number; row_count?: number;
error_message?: string; error_message?: string;
}; };
@@ -131,6 +133,33 @@ export type FinishIngestionRun = {
error_message?: string; error_message?: string;
}; };
export type RawArtifact = {
artifact_id: number;
run_id: number | null;
source_id: number | null;
source_name: string | null;
original_filename: string;
raw_uri: string;
sha256: string;
size_bytes: number;
mime_type: string | null;
uploaded_by: string;
notes: string;
created_at: string;
};
export type CreateRawArtifact = {
run_id?: number;
source_id?: number;
original_filename: string;
raw_uri: string;
sha256: string;
size_bytes: number;
mime_type?: string;
uploaded_by?: string;
notes?: string;
};
const API_BASE_URL = const API_BASE_URL =
process.env.NEXT_PUBLIC_API_BASE_URL?.replace(/\/$/, "") ?? "http://127.0.0.1:8080"; process.env.NEXT_PUBLIC_API_BASE_URL?.replace(/\/$/, "") ?? "http://127.0.0.1:8080";
@@ -197,6 +226,17 @@ export async function finishIngestionRun(
}); });
} }
export async function fetchRawArtifacts(): Promise<RawArtifact[]> {
return fetchJson("/api/v1/raw-artifacts");
}
export async function createRawArtifact(payload: CreateRawArtifact): Promise<RawArtifact> {
return fetchJson("/api/v1/raw-artifacts", {
method: "POST",
body: JSON.stringify(payload),
});
}
async function fetchJson<T>(path: string, init?: RequestInit): Promise<T> { async function fetchJson<T>(path: string, init?: RequestInit): Promise<T> {
const response = await fetch(`${API_BASE_URL}${path}`, { const response = await fetch(`${API_BASE_URL}${path}`, {
...init, ...init,

View File

@@ -46,6 +46,17 @@ PYTHONPATH=src python3 -m shanghai_housing validate-import \
通过时输出行数和模板名;失败时逐行返回错误信息。 通过时输出行数和模板名;失败时逐行返回错误信息。
## 原始文件指纹
导入前先计算文件指纹,用于登记 `audit.raw_artifacts` 并识别重复文件。
```bash
PYTHONPATH=src python3 -m shanghai_housing fingerprint-file \
--file docs/import_templates/area_monthly_metrics.csv
```
输出内容包含 `original_filename``raw_uri``sha256``size_bytes``mime_type`,可作为 `POST /api/v1/raw-artifacts` 的主要 payload。
## API 校验 ## API 校验
```bash ```bash
@@ -72,3 +83,22 @@ curl -X POST http://127.0.0.1:8080/api/v1/imports/validate \
] ]
}' }'
``` ```
## API 原始文件登记
```bash
curl -X POST http://127.0.0.1:8080/api/v1/raw-artifacts \
-H 'content-type: application/json' \
-d '{
"run_id": 1,
"source_id": 1,
"original_filename": "area_monthly_metrics.csv",
"raw_uri": "raw://sha256/<sha256>/area_monthly_metrics.csv",
"sha256": "<64-character-lowercase-sha256>",
"size_bytes": 2048,
"mime_type": "text/csv",
"uploaded_by": "researcher"
}'
```
同一个 `sha256` 重复登记时API 返回已有 artifact并可将新的导入批次关联到该 artifact。

View File

@@ -100,22 +100,45 @@
### M1.4 原始数据留存与文件哈希 ### M1.4 原始数据留存与文件哈希
状态:已完成。
目标: 目标:
- 每次导入都记录原始文件位置、哈希、大小、导入人和入库批次。 - 每次导入都记录原始文件位置、哈希、大小、导入人和入库批次。
- 正式导入结果写入 `raw`/`silver`/`audit`,并`audit.ingestion_runs` 关联。 - 将原始文件资产`audit.ingestion_runs` 关联,支持重复文件识别
交付物: 交付物:
- `audit.raw_artifacts` - `audit.raw_artifacts`
- 导入批次关联 raw artifact。 - 导入批次关联 raw artifact。
- PostgreSQL 导入执行接口。 - `GET /api/v1/raw-artifacts`
- `POST /api/v1/raw-artifacts`
- Python 文件指纹命令。
验收标准: 验收标准:
- 同一文件重复导入可识别。 - 同一文件重复导入可识别。
- 任一指标可以追溯到原始数据批次。 - 任一指标可以追溯到原始数据批次。
### M1.5 PostgreSQL 导入执行接口
目标:
- 将已校验模板数据正式写入 PostgreSQL 的 `raw`/`silver`/`audit`
- 导入执行必须关联 `audit.ingestion_runs``audit.raw_artifacts`
交付物:
- PostgreSQL 导入执行 API。
- 支持板块月度指标和小区月度指标的 upsert。
- 导入失败时写入明确错误并保留审计记录。
验收标准:
- 校验失败不写入业务表。
- 导入成功后可通过批次追溯到原始文件哈希。
- 重复导入同一文件时能识别并提示。
## M2 资产研究工作台 ## M2 资产研究工作台
### M2.1 板块详情页 ### M2.1 板块详情页

View File

@@ -7,6 +7,7 @@ from typing import Optional
from .db import DEFAULT_DB_PATH, connect, import_template_file, init_db, load_sample_data from .db import DEFAULT_DB_PATH, connect, import_template_file, init_db, load_sample_data
from .indicators import compute_area_scores from .indicators import compute_area_scores
from .import_templates import ImportTemplateError, validate_tabular_file from .import_templates import ImportTemplateError, validate_tabular_file
from .raw_artifacts import fingerprint_file
from .reporting import format_scores_table, write_monthly_report from .reporting import format_scores_table, write_monthly_report
@@ -69,6 +70,17 @@ def build_parser() -> argparse.ArgumentParser:
help="CSV, XLSX, or XLSM file to import.", help="CSV, XLSX, or XLSM file to import.",
) )
fingerprint_parser = subparsers.add_parser(
"fingerprint-file",
help="Compute raw artifact metadata for an import file.",
)
fingerprint_parser.add_argument(
"--file",
type=Path,
required=True,
help="File to hash and describe.",
)
return parser return parser
@@ -132,5 +144,10 @@ def main(argv: Optional[list[str]] = None) -> int:
) )
return 0 return 0
if args.command == "fingerprint-file":
fingerprint = fingerprint_file(args.file)
print(fingerprint.to_json())
return 0
parser.error(f"Unknown command: {args.command}") parser.error(f"Unknown command: {args.command}")
return 2 return 2

View File

@@ -0,0 +1,40 @@
from __future__ import annotations
import hashlib
import json
import mimetypes
from dataclasses import asdict, dataclass
from pathlib import Path
@dataclass(frozen=True)
class RawArtifactFingerprint:
original_filename: str
raw_uri: str
sha256: str
size_bytes: int
mime_type: str | None
def to_json(self) -> str:
return json.dumps(asdict(self), ensure_ascii=False, indent=2)
def fingerprint_file(file_path: Path) -> RawArtifactFingerprint:
path = file_path.resolve()
digest = hashlib.sha256()
size_bytes = 0
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
size_bytes += len(chunk)
sha256 = digest.hexdigest()
mime_type, _ = mimetypes.guess_type(path.name)
return RawArtifactFingerprint(
original_filename=path.name,
raw_uri=f"raw://sha256/{sha256}/{path.name}",
sha256=sha256,
size_bytes=size_bytes,
mime_type=mime_type,
)

View File

@@ -7,6 +7,7 @@ from pathlib import Path
from shanghai_housing.db import connect, import_template_file from shanghai_housing.db import connect, import_template_file
from shanghai_housing.import_templates import validate_csv_file, validate_import_rows from shanghai_housing.import_templates import validate_csv_file, validate_import_rows
from shanghai_housing.raw_artifacts import fingerprint_file
class ImportTemplateTests(unittest.TestCase): class ImportTemplateTests(unittest.TestCase):
@@ -79,6 +80,15 @@ class ImportTemplateTests(unittest.TestCase):
self.assertEqual(len(rows), 1) self.assertEqual(len(rows), 1)
self.assertEqual(rows[0]["level"], "shanghai") self.assertEqual(rows[0]["level"], "shanghai")
def test_fingerprints_import_file_without_local_path(self) -> None:
fingerprint = fingerprint_file(Path("docs/import_templates/policy_events.csv"))
self.assertEqual(fingerprint.original_filename, "policy_events.csv")
self.assertEqual(len(fingerprint.sha256), 64)
self.assertGreater(fingerprint.size_bytes, 0)
self.assertTrue(fingerprint.raw_uri.startswith("raw://sha256/"))
self.assertNotIn("/Users/", fingerprint.raw_uri)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()