AI/teaching-feedback-assistant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(observability): add structured application logging

Browse Source 
Emit correlated JSON logs from the API and FunASR services, propagate request IDs to the client, and configure bounded opt-in Docker logging.

Document the host-wide Grafana/Loki query workflow and keep observability deployment ownership outside the application repository.
This commit is contained in:
shay7sev
2026-07-22 13:52:40 +08:00
parent a63e9a1705
commit a875fe9f63
30 changed files with 1261 additions and 817 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
asr-service/Dockerfile
View File

@@ -2,10 +2,16 @@ FROM python:3.10-slim
ARG DEBIAN_MIRROR=http://deb.debian.org
ARG PYPI_INDEX_URL=https://pypi.org/simple
ARG APP_VERSION=0.2.0
ARG GIT_SHA=unknown
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1
PIP_NO_CACHE_DIR=1 \
APP_ENV=lan \
APP_VERSION=${APP_VERSION} \
GIT_SHA=${GIT_SHA} \
LOG_FORMAT=json
RUN sed -i "s|http://deb.debian.org|${DEBIAN_MIRROR}|g" /etc/apt/sources.list.d/debian.sources \
&& apt-get update \
@@ -18,7 +24,7 @@ RUN pip install --index-url "${PYPI_INDEX_URL}" --upgrade pip
RUN pip install torch==2.8.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cpu
RUN pip install --index-url "${PYPI_INDEX_URL}" -r requirements.txt
COPY app.py ./
COPY app.py logging_config.py ./
EXPOSE 10095
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "10095", "--workers", "1", "--no-access-log"]

146
asr-service/app.py
View File

@@ -4,16 +4,20 @@ import asyncio
import logging
import os
import tempfile
import time
import uuid
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any
from fastapi import FastAPI, File, Form, HTTPException, UploadFile
from fastapi import FastAPI, File, Form, HTTPException, Request, UploadFile
from funasr import AutoModel
from pydantic import BaseModel
LOGGER = logging.getLogger("funasr-service")
from logging_config import configure_logging, log_event, redirect_standard_streams
configure_logging()
MAX_AUDIO_BYTES = 15 * 1024 * 1024
CHUNK_BYTES = 1024 * 1024
SUPPORTED_FORMATS = {"mp3", "aac", "wav", "m4a", "amr"}
@@ -53,12 +57,12 @@ def positive_int(name: str, default: int) -> int:
def load_model(settings: Settings) -> Any:
LOGGER.info(
"loading FunASR model model=%s vad=%s punc=%s device=%s",
settings.model,
settings.vad_model,
settings.punc_model,
settings.device,
log_event(
logging.INFO,
"asr_model_loading",
"loading FunASR model",
model=settings.model,
device=settings.device,
)
model = AutoModel(
model=settings.model,
@@ -67,12 +71,13 @@ def load_model(settings: Settings) -> Any:
device=settings.device,
disable_update=True,
)
LOGGER.info("FunASR model is ready")
log_event(logging.INFO, "asr_model_ready", "FunASR model is ready")
return model
@asynccontextmanager
async def lifespan(app: FastAPI):
redirect_standard_streams()
settings = Settings()
app.state.settings = settings
app.state.model = await asyncio.to_thread(load_model, settings)
@@ -82,11 +87,31 @@ async def lifespan(app: FastAPI):
app = FastAPI(
title="Teaching Feedback FunASR Service",
version="1.0.0",
version=os.getenv("APP_VERSION", "0.2.0"),
lifespan=lifespan,
)
@app.middleware("http")
async def request_id_middleware(request: Request, call_next: Any):
request_id = valid_request_id(request.headers.get("x-request-id"))
request.state.request_id = request_id
try:
response = await call_next(request)
except Exception:
log_event(
logging.ERROR,
"asr_http_request_failed",
"ASR HTTP request failed",
request_id=request_id,
status=500,
error_class="unhandled_error",
)
raise
response.headers["X-Request-ID"] = request_id
return response
@app.get("/health", response_model=HealthResponse)
async def health() -> HealthResponse:
settings: Settings = app.state.settings
@@ -95,17 +120,32 @@ async def health() -> HealthResponse:
@app.post("/v1/transcriptions", response_model=TranscriptionResponse)
async def create_transcription(
request: Request,
file: UploadFile = File(...),
language: str = Form("zh"),
duration_ms: int = Form(0),
hotword: str = Form(""),
) -> TranscriptionResponse:
if language not in {"zh", "zh-cn", "cmn"}:
raise HTTPException(status_code=422, detail="this service is configured for Chinese audio")
audio_format = file_extension(file.filename)
temporary_path = await save_upload(file, audio_format)
request_id = str(uuid.uuid4())
request_id = request.state.request_id
started_at = time.perf_counter()
audio_format = "unknown"
audio_size_bytes = 0
temporary_path: Path | None = None
normalized_duration_ms = max(0, duration_ms)
log_event(
logging.INFO,
"asr_request_started",
"ASR request started",
request_id=request_id,
audio_duration_ms=normalized_duration_ms,
audio_format=audio_format,
)
try:
if language not in {"zh", "zh-cn", "cmn"}:
raise HTTPException(status_code=422, detail="this service is configured for Chinese audio")
audio_format = file_extension(file.filename)
temporary_path, audio_size_bytes = await save_upload(file, audio_format)
async with app.state.inference_slots:
text = await asyncio.to_thread(
run_inference,
@@ -114,21 +154,79 @@ async def create_transcription(
app.state.settings.batch_size_seconds,
hotword.strip(),
)
except Exception as error:
LOGGER.exception("FunASR inference failed request_id=%s", request_id)
raise HTTPException(status_code=500, detail=f"speech recognition failed: {error}") from error
if not text:
raise HTTPException(status_code=422, detail="no clear speech was recognized")
except HTTPException as error:
log_event(
logging.WARNING,
"asr_request_failed",
"ASR request failed",
request_id=request_id,
status=error.status_code,
latency_ms=elapsed_ms(started_at),
audio_size_bytes=audio_size_bytes,
audio_duration_ms=normalized_duration_ms,
audio_format=audio_format,
error_class=http_error_class(error.status_code),
)
raise
except Exception:
log_event(
logging.ERROR,
"asr_request_failed",
"ASR request failed",
request_id=request_id,
status=500,
latency_ms=elapsed_ms(started_at),
audio_size_bytes=audio_size_bytes,
audio_duration_ms=normalized_duration_ms,
audio_format=audio_format,
error_class="inference_failed",
)
raise HTTPException(status_code=500, detail="speech recognition failed") from None
finally:
temporary_path.unlink(missing_ok=True)
if temporary_path is not None:
temporary_path.unlink(missing_ok=True)
if not text:
raise HTTPException(status_code=422, detail="no clear speech was recognized")
log_event(
logging.INFO,
"asr_request_finished",
"ASR request finished",
request_id=request_id,
status=200,
latency_ms=elapsed_ms(started_at),
audio_size_bytes=audio_size_bytes,
audio_duration_ms=normalized_duration_ms,
audio_format=audio_format,
)
return TranscriptionResponse(
text=text,
duration_ms=max(0, duration_ms),
duration_ms=normalized_duration_ms,
request_id=request_id,
)
def valid_request_id(value: str | None) -> str:
candidate = (value or "").strip()
if candidate and len(candidate) <= 128 and candidate.isascii() and candidate.isprintable():
return candidate
return str(uuid.uuid4())
def elapsed_ms(started_at: float) -> int:
return int((time.perf_counter() - started_at) * 1000)
def http_error_class(status: int) -> str:
if status == 413:
return "audio_too_large"
if status == 415:
return "unsupported_audio_format"
if status == 422:
return "invalid_audio"
return "request_error"
def file_extension(filename: str | None) -> str:
suffix = Path(filename or "segment.mp3").suffix.lower().lstrip(".")
if suffix not in SUPPORTED_FORMATS:
@@ -136,7 +234,7 @@ def file_extension(filename: str | None) -> str:
return suffix
async def save_upload(file: UploadFile, audio_format: str) -> Path:
async def save_upload(file: UploadFile, audio_format: str) -> tuple[Path, int]:
handle = tempfile.NamedTemporaryFile(suffix=f".{audio_format}", delete=False)
path = Path(handle.name)
total_bytes = 0
@@ -155,7 +253,7 @@ async def save_upload(file: UploadFile, audio_format: str) -> Path:
if total_bytes == 0:
path.unlink(missing_ok=True)
raise HTTPException(status_code=422, detail="audio file is empty")
return path
return path, total_bytes
def run_inference(model: Any, path: Path, batch_size_seconds: int, hotword: str) -> str:

90
asr-service/logging_config.py Normal file
View File

@@ -0,0 +1,90 @@
from __future__ import annotations
import json
import logging
import os
import sys
from datetime import datetime, timezone
from typing import Any
LOG_FIELDS = (
"event",
"request_id",
"segment_id",
"feedback_session_id",
"status",
"latency_ms",
"audio_size_bytes",
"audio_duration_ms",
"audio_format",
"error_class",
)
class JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
payload: dict[str, Any] = {
"timestamp": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),
"level": record.levelname,
"service": "funasr",
"environment": os.getenv("APP_ENV", "development"),
"app_version": os.getenv("APP_VERSION", "unknown"),
"git_sha": os.getenv("GIT_SHA", "unknown"),
"message": record.getMessage(),
}
for field in LOG_FIELDS:
value = getattr(record, field, None)
if value is not None:
payload[field] = value
return json.dumps(payload, ensure_ascii=False, default=str)
class JsonLogStream:
encoding = "utf-8"
def __init__(self, level: int) -> None:
self.level = level
self.buffer = ""
def write(self, message: str) -> int:
self.buffer += message
while "\n" in self.buffer:
line, self.buffer = self.buffer.split("\n", 1)
self._emit(line)
return len(message)
def flush(self) -> None:
if self.buffer:
self._emit(self.buffer)
self.buffer = ""
def isatty(self) -> bool:
return False
def _emit(self, message: str) -> None:
message = message.strip()
if message:
logging.getLogger("funasr-service").log(self.level, message)
def configure_logging() -> None:
handler = logging.StreamHandler(sys.__stdout__)
handler.setFormatter(JsonFormatter())
root = logging.getLogger()
root.handlers.clear()
root.addHandler(handler)
root.setLevel(logging.INFO)
for name, logger in logging.Logger.manager.loggerDict.items():
if isinstance(logger, logging.Logger):
logger.handlers.clear()
logger.propagate = name != "uvicorn.access"
def redirect_standard_streams() -> None:
sys.stdout = JsonLogStream(logging.INFO) # type: ignore[assignment]
sys.stderr = JsonLogStream(logging.ERROR) # type: ignore[assignment]
def log_event(level: int, event: str, message: str, **fields: Any) -> None:
logging.getLogger("funasr-service").log(level, message, extra={"event": event, **fields})