Руководство по мониторингу

from src.monitoring import (
    MetricsCollector,
    StructuredLogger,
    monitor_scenario,
    monitor_agent,
    monitor_cache,
    record_llm_call,
)

# Создание структурированного логера
logger = StructuredLogger("my_component")

# Получение сборщика метрик (синглтон)
metrics = MetricsCollector()

# Использование декораторов для автоматического отслеживания
@monitor_scenario("my_scenario")
def process_query(query: str):
    # Автоматически записывает SCENARIO_DURATION, SCENARIO_SUCCESS/FAILURE,
    # ACTIVE_REQUESTS, TOTAL_REQUESTS
    return result

from src.monitoring.health import create_health_app

# Создание standalone FastAPI-приложения для проверки работоспособности
app = create_health_app()

# Доступные конечные точки:
# GET /health       - Общий статус работоспособности
# GET /health/live  - Проверка активности
# GET /health/ready - Проверка готовности
# GET /metrics      - Метрики Prometheus
# GET /stats        - Статистика системы

# SCENARIO_DURATION (секунды)
(0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 120.0)

# AGENT_DURATION (секунды)
(0.1, 0.5, 1.0, 2.0, 5.0, 10.0)

# LLM_LATENCY (секунды)
(0.1, 0.5, 1.0, 2.0, 3.0, 5.0, 10.0)

# CPG_QUERY_LATENCY (секунды)
(0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0)

# CPG_QUERY_RESULTS (количество)
(0, 1, 5, 10, 50, 100, 500, 1000)

# RETRIEVAL_LATENCY (секунды)
(0.1, 0.5, 1.0, 2.0, 5.0)

# RETRIEVAL_RESULTS (количество)
(0, 1, 5, 10, 20, 50)

from src.monitoring.metrics import (
    SCENARIO_DURATION,
    SCENARIO_SUCCESS,
    SCENARIO_FAILURE,
    LLM_LATENCY,
    LLM_TOKENS,
    LLM_ERRORS,
    CACHE_HITS,
    CACHE_MISSES,
)

# Запись времени выполнения сценария
with SCENARIO_DURATION.labels(scenario_name="security_audit").time():
    result = run_scenario()

SCENARIO_SUCCESS.labels(scenario_name="security_audit").inc()

# Запись использования LLM
LLM_LATENCY.labels(model="GigaChat-2-Pro", operation="generate").observe(2.5)
LLM_TOKENS.labels(model="GigaChat-2-Pro", token_type="output").inc(150)
LLM_ERRORS.labels(model="GigaChat-2-Pro", error_type="timeout").inc()

# Запись обращений к кешу
CACHE_HITS.labels(cache_type="query_plan").inc()
CACHE_MISSES.labels(cache_type="embedding").inc()

from src.monitoring import record_llm_call, record_cpg_query, record_retrieval

# Запись вызова LLM (обновляет LLM_LATENCY, LLM_TOKENS, LLM_ERRORS)
record_llm_call(
    model="GigaChat-2-Pro",
    operation="generate",
    duration=2.5,
    input_tokens=100,
    output_tokens=150,
    error=None,  # или строка с типом ошибки при сбое
)

# Запись CPG-запроса (обновляет CPG_QUERY_LATENCY, CPG_QUERY_RESULTS)
record_cpg_query(
    query_type="method_lookup",
    duration=0.05,
    result_count=12,
)

# Запись операции поиска (обновляет RETRIEVAL_LATENCY, RETRIEVAL_RESULTS)
record_retrieval(
    retrieval_type="hybrid",
    duration=1.2,
    result_count=10,
)

from src.monitoring import monitor_scenario

@monitor_scenario("security_audit")
def run_security_audit(codebase: str):
    # Записывает:
    # - SCENARIO_DURATION.labels(scenario_name="security_audit")
    # - SCENARIO_SUCCESS / SCENARIO_FAILURE
    # - ACTIVE_REQUESTS inc/dec
    # - TOTAL_REQUESTS inc
    return findings

from src.monitoring import monitor_agent

@monitor_agent("retriever_agent", scenario="security")
def retrieve_context(query: str) -> list:
    # Записывает:
    # - AGENT_DURATION.labels(agent_name="retriever_agent", scenario="security")
    # - AGENT_SUCCESS / AGENT_FAILURE
    return results

from src.monitoring import monitor_cache

@monitor_cache("query_plan")
def get_cached_query(key: str) -> dict | None:
    # Записывает:
    # - CACHE_HITS.labels(cache_type="query_plan") при непустом возврате
    # - CACHE_MISSES.labels(cache_type="query_plan") при возврате None
    return cached_result

from src.monitoring import MetricsCollector, get_metrics_collector

# Оба возвращают один и тот же экземпляр-синглтон
collector = MetricsCollector()
collector = get_metrics_collector()

# Запись метрик
collector.record_latency("query_generation", 0.245)
collector.record_scenario("security_audit", success=True, latency=1.5)
collector.record_cache_access("query_plan", hit=True)
collector.increment_counter("total_queries", amount=1)
collector.set_gauge("active_requests", 5.0)

# Получение сводки
summary = collector.get_summary()  # Возвращает MetricsSummary
stats = collector.get_scenario_stats()  # Статистика по сценариям
cache = collector.get_cache_stats()  # Статистика по типам кеша
uptime = collector.get_uptime_seconds()

# Сброс всех собранных метрик
collector.reset()

summary = collector.get_summary()
data = summary.to_dict()  # Преобразование в словарь для JSON-сериализации

from src.monitoring.health import HealthStatus

class HealthStatus(Enum):
    HEALTHY = "healthy"
    DEGRADED = "degraded"
    UNHEALTHY = "unhealthy"

from src.monitoring.health import HealthChecker, ComponentHealth, HealthStatus

checker = HealthChecker()

def check_custom_service():
    try:
        response_time = ping_service()
        if response_time < 100:
            return ComponentHealth(
                name="custom_service",
                status=HealthStatus.HEALTHY,
                latency_ms=response_time
            )
        else:
            return ComponentHealth(
                name="custom_service",
                status=HealthStatus.DEGRADED,
                latency_ms=response_time,
                message="Высокая задержка"
            )
    except Exception as e:
        return ComponentHealth(
            name="custom_service",
            status=HealthStatus.UNHEALTHY,
            message=str(e)
        )

checker.register_check("custom_service", check_custom_service)

{
  "status": "healthy",
  "timestamp": 1702580000.0,
  "uptime_seconds": 3600.0,
  "version": "2.0.0",
  "components": [
    {
      "name": "database",
      "status": "healthy",
      "latency_ms": 2.5,
      "message": "Connected, 1500 methods",
      "details": {}
    },
    {
      "name": "llm",
      "status": "healthy",
      "latency_ms": 450.0,
      "message": "LLM responding normally",
      "details": {"provider": "yandex"}
    }
  ]
}

{"status": "alive"}

{"status": "ready"}

{
  "summary": {"total_requests": 100, "success_rate": 0.95, "...": "..."},
  "scenarios": {"security_audit": {"total_requests": 50, "success_rate": 0.98}},
  "cache": {"query_plan": {"hits": 80, "misses": 20, "hit_rate": 0.8}},
  "uptime_seconds": 3600.0
}

{
  "status": "healthy",
  "version": "2.0.0",
  "uptime_seconds": 3600.0,
  "timestamp": "2026-03-07T10:30:00Z",
  "components": {
    "database": {"status": "healthy"},
    "llm": {"status": "healthy", "provider": "yandex"},
    "chromadb": {"status": "healthy", "collections": 5},
    "cpg": {"status": "healthy", "db_path": "/data/project.duckdb", "methods": 1500}
  }
}

from src.monitoring.health import (
    check_database_connection,
    check_llm_availability,
    check_vector_store,
    run_health_check_cli,
)

# Быстрые логические проверки
db_ok = check_database_connection()    # True, если DuckDB отвечает на SELECT 1
llm_ok = check_llm_availability()      # True, если LLM is_available()
vs_ok = check_vector_store()           # True, если ChromaDB heartbeat успешен

# Проверка работоспособности из CLI (выводит отчёт, завершается с кодом 0/1/2)
run_health_check_cli()

from src.monitoring import StructuredLogger

logger = StructuredLogger("my_component")

# Логирование со структурированными данными
logger.info("Processing request", request_id="abc123", user="test")
logger.warning("Slow query", duration_ms=500)
logger.error("Failed to connect", service="database", error="timeout")

{
  "timestamp": "2026-03-07T10:30:00.000Z",
  "level": "INFO",
  "logger": "my_component",
  "message": "Processing request",
  "request_id": "abc123",
  "user": "test"
}

from src.monitoring import StructuredLogger

logger = StructuredLogger("query_engine")

with logger.timed_operation("query_generation", scenario="security"):
    result = generate_query()
    # При успехе логирует:
    # {"message": "Operation completed: query_generation",
    #  "operation": "query_generation", "duration_ms": 245.0,
    #  "status": "success", "scenario": "security"}

    # При исключении логирует ошибку с status="error", error_type, error_message
    # и пробрасывает исключение дальше

from src.api.middleware.metrics import MetricsMiddleware

app.add_middleware(MetricsMiddleware)

rate(rag_scenario_success_total[5m]) + rate(rag_scenario_failure_total[5m])

rate(rag_scenario_failure_total[5m]) / (rate(rag_scenario_success_total[5m]) + rate(rag_scenario_failure_total[5m]))

histogram_quantile(0.95, rate(rag_scenario_duration_seconds_bucket[5m]))

sum(rate(rag_llm_tokens_total[1h])) by (model)

rate(rag_cache_hits_total[5m]) / (rate(rag_cache_hits_total[5m]) + rate(rag_cache_misses_total[5m]))

sum(rate(rag_llm_errors_total[5m])) by (model, error_type)

rag_active_requests

groups:
  - name: codegraph_availability
    rules:
      - alert: CodeGraphAPIDown
        expr: up{job="codegraph-api"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "CodeGraph API недоступен"

      - alert: CodeGraphHighErrorRate
        expr: |
          (
            sum(rate(rag_scenario_failure_total[5m]))
            / (sum(rate(rag_scenario_success_total[5m])) + sum(rate(rag_scenario_failure_total[5m])))
          ) > 0.25
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Высокая частота ошибок сценариев (>25%)"

  - name: codegraph_latency
    rules:
      - alert: CodeGraphHighLatency
        expr: histogram_quantile(0.95, sum(rate(rag_scenario_duration_seconds_bucket[5m])) by (le)) > 30
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Высокая задержка сценариев (p95 > 30 с)"

      - alert: CodeGraphLLMSlowResponses
        expr: histogram_quantile(0.95, sum(rate(rag_llm_latency_seconds_bucket[5m])) by (le)) > 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Высокая задержка ответов LLM (p95 > 10 с)"

  - name: codegraph_resources
    rules:
      - alert: CodeGraphHighActiveRequests
        expr: rag_active_requests > 50
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "Большое количество активных запросов (>50)"

      - alert: CodeGraphLLMErrors
        expr: sum(rate(rag_llm_errors_total[5m])) > 0.1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Обнаружены ошибки LLM"

  - name: codegraph_infrastructure
    rules:
      - alert: PrometheusTargetDown
        expr: up == 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: "Цель Prometheus {{ $labels.job }} недоступна"

apiVersion: apps/v1
kind: Deployment
metadata:
  name: codegraph
spec:
  template:
    spec:
      containers:
        - name: codegraph
          ports:
            - containerPort: 8000
              name: api
          livenessProbe:
            httpGet:
              path: /api/v1/health/live
              port: api
            initialDelaySeconds: 30
            periodSeconds: 10
          readinessProbe:
            httpGet:
              path: /api/v1/health/ready
              port: api
            initialDelaySeconds: 10
            periodSeconds: 5

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: codegraph
spec:
  selector:
    matchLabels:
      app: codegraph
  endpoints:
    - port: api
      path: /api/v1/metrics
      interval: 30s

/api/v1/metrics

groups:
  - name: codegraph_dashboard
    rules:
      - alert: DashboardProjectRiskCritical
        expr: codegraph_project_health_score < 30
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Project {{ $labels.project }} health score critically low ({{ $value }})"
          runbook: "Check audit, SCA, and compliance status. Escalate to team lead."

      - alert: DashboardProjectRiskHigh
        expr: codegraph_project_health_score < 50
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "Project {{ $labels.project }} in red zone (health {{ $value }})"

      - alert: DashboardComplianceGap
        expr: codegraph_project_compliance_score < 40
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "Project {{ $labels.project }} compliance score below 40% ({{ $value }})"

      - alert: DashboardReleaseGateFail
        expr: codegraph_project_release_status == 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Project {{ $labels.project }} release gate blocked"

      - alert: DashboardScaCriticalVuln
        expr: codegraph_project_sca_vulnerabilities{severity="critical"} > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Project {{ $labels.project }} has {{ $value }} critical SCA vulnerabilities"

# Проекты с оценкой ниже 50 (красная зона)
codegraph_project_health_score < 50

# Средняя оценка по всем проектам в группе
avg(codegraph_project_health_score{group="backend"})

# 5 худших проектов по оценке аудита
bottomk(5, codegraph_project_audit_score)

# Проекты с соответствием ниже 40%
codegraph_project_compliance_score < 40

# Количество проектов со статусом fail
count(codegraph_project_release_status == 0)

# Проекты с SCA-оценкой ниже 70
codegraph_project_sca_health < 70

# Общее количество критических уязвимостей по всем проектам
sum(codegraph_project_sca_vulnerabilities{severity="critical"})

# Количество уязвимостей по каждому проекту
sum by (project) (codegraph_project_sca_vulnerabilities)

# Работоспособность портфеля по всем группам
codegraph_portfolio_avg_health{group=""}

# Работоспособность по группе
codegraph_portfolio_avg_health{group="backend"}

# Оповещение при наличии release-related red-zone items
sum(codegraph_red_zone_items_count{category="release"}) > 0

GET /api/v1/metrics

# HELP codegraph_project_health_score Composite health score for a project
# TYPE codegraph_project_health_score gauge
codegraph_project_health_score{project="payments-api",group="backend"} 85.0
codegraph_project_health_score{project="legacy-auth",group="backend"} 35.0

# HELP codegraph_portfolio_avg_health Average health score across all projects
# TYPE codegraph_portfolio_avg_health gauge
codegraph_portfolio_avg_health{group=""} 72.5
codegraph_portfolio_avg_health{group="backend"} 60.0

# HELP codegraph_red_zone_items_count Number of red zone items by project and category
# TYPE codegraph_red_zone_items_count gauge
codegraph_red_zone_items_count{project="legacy-auth",category="release"} 2

scrape_configs:
  - job_name: codegraph-dashboard
    scrape_interval: 5m
    metrics_path: /api/v1/metrics
    static_configs:
      - targets: ["localhost:8000"]