diff --git a/README.md b/README.md index 25b43c8..9356a35 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,13 @@ same database. Your scripts and admin tools can still write it. cfgit sits besid the store, records what changed, and refuses to clobber changes it did not record.

- Side-by-side line diff of a live agent config, with a sticky field header and collapsed context - System-impact panel: deterministic facts plus opt-in LLM narration of what the change does downstream - Scoped impact: select records on the left and reason the change against only those + Production-shaped support agent drift with collection stats, branch workflow controls, history, and line-aligned diff + High-risk system-impact panel for a refund-resolution agent change with provider-backed LLM narration + Scoped impact analysis with selected policy, eval, and rollout records plus provider-backed narration

- Line-aligned diff of a live record  ·  system-impact panel  ·  impact scoped to the records you select  (demo data) + Production-shaped drift review  ·  narrated whole-system impact  ·  narrated impact scoped to selected policy, eval, and rollout records

## Why cfgit exists @@ -357,11 +357,11 @@ If you omit `--port`, cfgit will try the next free local ports. If you pass `--port` explicitly, cfgit treats that port as intentional and fails if it is already in use. -## Synthetic UI demo +## Production-shaped UI demo -The repo includes a fake support-control-plane fixture for screenshots, demos, -and UI testing. It creates only synthetic records in the database you pass; use a -throwaway database name when you want to keep the run contained. +The repo includes a safe support-control-plane fixture for screenshots, demos, +and UI testing. It creates synthetic records only in the database you pass; use a +throwaway database name when you want the run contained. ```bash python examples/seed_support_demo.py \ @@ -369,22 +369,26 @@ python examples/seed_support_demo.py \ --db cfgit_ui_demo \ --reset -cfg --config-file examples/cfgit-support-demo.toml init -cfg --config-file examples/cfgit-support-demo.toml import --all -m "initial synthetic demo import" +cfg --config-file examples/cfgit-support-demo.toml --env dev init +cfg --config-file examples/cfgit-support-demo.toml --env dev import --all -m "initial production-shaped demo import" python examples/seed_support_demo.py \ --uri 'mongodb://localhost:27017/?replicaSet=rs0' \ --db cfgit_ui_demo \ --drift -cfg --config-file examples/cfgit-support-demo.toml ui +cfg --config-file examples/cfgit-support-demo.toml --env dev status +cfg --config-file examples/cfgit-support-demo.toml --env dev ui ``` -The first seed creates clean planner, critic, router, model, and policy records. -The `--drift` pass then simulates an admin-console edit: planner routing changes, -the refund policy changes, and a new entitlements resolver appears. That gives -the UI useful drift, impact, adopt, branch, PR, merge, and restore-history paths -without using proprietary data. +The base seed creates a realistic support-agent runtime surface: orchestrator +and specialist agent configs, model routing, policy rules, tool registry records, +routing policies, escalation policies, eval suites, rollout controls, and +knowledge-source allowlists. The `--drift` pass simulates admin-console edits to +refund automation, refund policy, routing confidence, eval gates, rollout +traffic, and a new loyalty-credit tool. That gives the UI useful drift, impact, +scoped impact, adopt, branch, PR, merge, and restore-history paths without using +proprietary data. ## MCP and agent usage diff --git a/docs/screenshots/01-diff.png b/docs/screenshots/01-diff.png index 9d7aebb..9491766 100644 Binary files a/docs/screenshots/01-diff.png and b/docs/screenshots/01-diff.png differ diff --git a/docs/screenshots/02-impact.png b/docs/screenshots/02-impact.png index b82cf94..f357104 100644 Binary files a/docs/screenshots/02-impact.png and b/docs/screenshots/02-impact.png differ diff --git a/docs/screenshots/03-scoped-impact.png b/docs/screenshots/03-scoped-impact.png index 4857e8e..1766d60 100644 Binary files a/docs/screenshots/03-scoped-impact.png and b/docs/screenshots/03-scoped-impact.png differ diff --git a/examples/cfgit-support-demo.toml b/examples/cfgit-support-demo.toml index c239aa6..8b186e0 100644 --- a/examples/cfgit-support-demo.toml +++ b/examples/cfgit-support-demo.toml @@ -31,6 +31,48 @@ live_when = { active = true } ignore_fields = ["_id", "active", "updated_at", "updated_by"] secret_fields = [] +[[collection]] +name = "tool_registry" +id_field = "tool_id" +live_when = { enabled = true } +ignore_fields = ["_id", "enabled", "updated_at", "updated_by"] +secret_fields = ["credentials.api_key_ref", "oauth.client_secret_ref"] + +[[collection]] +name = "routing_policies" +id_field = "policy_id" +live_when = { active = true } +ignore_fields = ["_id", "active", "updated_at", "updated_by"] +secret_fields = [] + +[[collection]] +name = "escalation_policies" +id_field = "escalation_id" +live_when = { active = true } +ignore_fields = ["_id", "active", "updated_at", "updated_by"] +secret_fields = [] + +[[collection]] +name = "eval_suites" +id_field = "suite_id" +live_when = { enabled = true } +ignore_fields = ["_id", "enabled", "updated_at", "updated_by"] +secret_fields = [] + +[[collection]] +name = "rollout_controls" +id_field = "rollout_id" +live_when = { enabled = true } +ignore_fields = ["_id", "enabled", "updated_at", "updated_by"] +secret_fields = [] + +[[collection]] +name = "knowledge_sources" +id_field = "source_id" +live_when = { enabled = true } +ignore_fields = ["_id", "enabled", "updated_at", "updated_by"] +secret_fields = [] + [secrets] block_fields = ["*_key", "*_secret", "*_token", "*api_key*", "*password*"] block_values = ["sk-[A-Za-z0-9]{20,}", "AKIA[0-9A-Z]{16}"] @@ -45,10 +87,24 @@ share_with_ai = [] ai_provider = "openai" warn_level = "none" links = [ - { field = "phase_contract", means = "contract other records may rely on" }, - { field = "tools", means = "shared tool list" }, - { field = "fallback_models", means = "fallback model routing" }, + { field = "agent", means = "agent identifier" }, + { field = "agents", means = "agent identifiers" }, { field = "applies_to", means = "policy-to-agent linkage" }, + { field = "approval_policy", means = "human approval requirement" }, + { field = "audience", means = "customer or segment targeting" }, + { field = "escalation_policy", means = "human handoff policy" }, + { field = "eval_suite", means = "quality gate for a config" }, + { field = "fallback_models", means = "fallback model routing" }, + { field = "guarded_by", means = "policy or guardrail dependency" }, + { field = "handoffs", means = "agent handoff graph" }, + { field = "knowledge_sources", means = "retrieval source allowlist" }, + { field = "model", means = "model routing" }, + { field = "model_path", means = "model identifier" }, + { field = "policy_refs", means = "referenced policy rules" }, + { field = "routing_policy", means = "orchestration routing policy" }, + { field = "rollout_id", means = "rollout gate" }, + { field = "tool_id", means = "tool identifier" }, + { field = "tools", means = "shared tool list" }, ] [env.dev] diff --git a/examples/seed_support_demo.py b/examples/seed_support_demo.py index 4749d4c..e1b4cd3 100644 --- a/examples/seed_support_demo.py +++ b/examples/seed_support_demo.py @@ -1,7 +1,9 @@ -"""Seed a synthetic cfgit demo database. +"""Seed a synthetic but production-shaped cfgit demo database. -This fixture is intentionally fake: support agents, model routes, and policy -rules for demo screenshots. It never reads production data. +The data is intentionally fake and safe for screenshots. It is modeled after +public patterns for agent handoffs, guardrails, escalation flows, evals, tracing, +knowledge sources, and rollout gates, but it does not copy a vendor config or any +private customer setup. Typical demo flow: python examples/seed_support_demo.py --reset @@ -13,10 +15,29 @@ import argparse from datetime import datetime, timezone +from typing import Any from pymongo import MongoClient +RUNTIME_COLLECTIONS = [ + "agent_configs", + "modelgarden_models", + "policy_rules", + "tool_registry", + "routing_policies", + "escalation_policies", + "eval_suites", + "rollout_controls", + "knowledge_sources", +] +CFGIT_COLLECTIONS = [ + "cfgit_demo_history", + "cfgit_demo_heads", + "cfgit_demo_refs", +] + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--uri", default="mongodb://localhost:27017/?replicaSet=rs0") @@ -34,14 +55,7 @@ def main() -> None: if args.db in {"admin", "config", "local"}: raise SystemExit(f"refusing to seed Mongo system database {args.db!r}") if args.reset: - for name in [ - "agent_configs", - "modelgarden_models", - "policy_rules", - "cfgit_demo_history", - "cfgit_demo_heads", - "cfgit_demo_refs", - ]: + for name in [*RUNTIME_COLLECTIONS, *CFGIT_COLLECTIONS]: db.drop_collection(name) now = datetime.now(timezone.utc) @@ -54,175 +68,1044 @@ def main() -> None: print(f"seeded synthetic cfgit demo data in Mongo database {args.db!r}") -def seed_base(db, now: datetime) -> None: - db.agent_configs.insert_many( +def seed_base(db: Any, now: datetime) -> None: + _replace_many( + db.agent_configs, + "config_id", [ { - "config_id": "planner", + "config_id": "support_orchestrator", + "is_active": True, + "role": "Support Orchestrator", + "model": "openai/gpt-4.1-mini", + "fallback_models": ["anthropic/claude-3-5-sonnet"], + "routing_policy": "global_support_router", + "escalation_policy": "human_handoff_standard", + "eval_suite": "support_orchestrator_regression", + "rollout_id": "support_orchestrator_june", + "tools": ["ticket_lookup", "customer_profile", "kb_search"], + "knowledge_sources": ["kb_public_help_center", "kb_internal_runbooks"], + "handoffs": ["refund_resolution", "billing_disputes", "delivery_incidents"], + "guarded_by": [ + "privacy_minimization_v2", + "refund_window_standard", + "regulated_advice_boundary", + ], + "policy_refs": ["privacy_minimization_v2", "refund_window_standard"], + "automation_threshold": 0.74, + "max_turns": 9, + "trace_sample_rate": 0.15, + "phase_contract": ( + "Classify the customer intent, gather missing identifiers, select the " + "specialist agent, and preserve enough context for audit review." + ), + "instructions": ( + "Start every run by reading ticket_lookup and customer_profile. Route " + "refunds to refund_resolution, billing failures to billing_disputes, " + "and carrier exceptions to delivery_incidents. Do not promise credits, " + "legal conclusions, medical advice, or policy exceptions. Escalate when " + "confidence is below automation_threshold or a guardrail fires." + ), + "updated_at": now, + "updated_by": "seed", + }, + { + "config_id": "refund_resolution", + "is_active": True, + "role": "Refund Resolution Specialist", + "model": "openai/gpt-4.1-mini", + "fallback_models": ["openai/gpt-4.1"], + "routing_policy": "refund_router", + "escalation_policy": "payments_handoff_high_risk", + "eval_suite": "refund_policy_regression", + "rollout_id": "refund_agent_june", + "tools": ["order_ledger", "payment_refund_quote", "ticket_update"], + "knowledge_sources": ["kb_public_refunds", "kb_internal_payments_runbook"], + "handoffs": ["billing_disputes", "trust_safety_review"], + "guarded_by": [ + "refund_window_standard", + "chargeback_no_refund_v1", + "privacy_minimization_v2", + ], + "policy_refs": ["refund_window_standard", "chargeback_no_refund_v1"], + "approval_policy": "supervisor_approval_over_250", + "automation_threshold": 0.81, + "max_credit_usd": 250, + "trace_sample_rate": 0.25, + "phase_contract": ( + "Determine refund eligibility, quote an auditable refund amount, and " + "write the customer-facing rationale before any payment mutation." + ), + "instructions": ( + "Use order_ledger before refund_quote. Apply refund_window_standard and " + "deny refunds when chargeback_no_refund_v1 applies. If the amount exceeds " + "max_credit_usd or abuse signals are present, hand off to a supervisor." + ), + "updated_at": now, + "updated_by": "seed", + }, + { + "config_id": "billing_disputes", "is_active": True, - "role": "Support Triage Planner", - "model": "openai/gpt-4o-mini", - "tools": ["search", "calendar", "handoff"], - "fallback_models": ["anthropic/claude-haiku"], + "role": "Billing Dispute Analyst", + "model": "anthropic/claude-3-5-sonnet", + "fallback_models": ["openai/gpt-4.1"], + "routing_policy": "billing_router", + "escalation_policy": "payments_handoff_high_risk", + "eval_suite": "billing_dispute_regression", + "rollout_id": "billing_disputes_june", + "tools": ["invoice_lookup", "payment_refund_quote", "ticket_update"], + "knowledge_sources": ["kb_internal_payments_runbook"], + "handoffs": ["trust_safety_review"], + "guarded_by": ["chargeback_no_refund_v1", "privacy_minimization_v2"], + "policy_refs": ["chargeback_no_refund_v1", "privacy_minimization_v2"], + "approval_policy": "finance_ops_approval", + "automation_threshold": 0.86, "phase_contract": ( - "Produces a support-ticket plan with risk, refund-policy, " - "and owner handoff notes." + "Separate billing-system errors from customer misunderstanding, then " + "prepare the adjustment rationale for finance review." ), "instructions": ( - "Classify incoming support tickets, cite the current refund " - "policy, and hand off risky cases." + "Never reverse a charge directly. Gather invoice id, payment processor " + "id, country, and dispute reason. Escalate active chargebacks, tax issues, " + "and invoice corrections above 500 USD." ), "updated_at": now, "updated_by": "seed", }, { - "config_id": "critic", + "config_id": "delivery_incidents", "is_active": True, - "role": "Policy Reviewer", - "model": "anthropic/claude-haiku", - "tools": ["diff", "policy_check"], + "role": "Delivery Incident Coordinator", + "model": "openai/gpt-4.1-mini", + "fallback_models": ["anthropic/claude-3-5-sonnet"], + "routing_policy": "logistics_router", + "escalation_policy": "operations_handoff", + "eval_suite": "delivery_exception_regression", + "rollout_id": "delivery_incidents_june", + "tools": ["shipment_trace", "warehouse_inventory", "ticket_update"], + "knowledge_sources": ["kb_public_shipping", "kb_internal_ops_runbook"], + "handoffs": ["refund_resolution"], + "guarded_by": ["replacement_fraud_boundary", "privacy_minimization_v2"], + "policy_refs": ["replacement_fraud_boundary"], + "automation_threshold": 0.79, "phase_contract": ( - "Reviews planner decisions for refund-policy compliance and " - "escalation risk." + "Identify shipment state, choose replacement/refund/no-action path, and " + "attach carrier evidence to the ticket." ), "instructions": ( - "Review planner output from planner, verify refund_window_v1, " - "and flag missing restore notes." + "Use shipment_trace before warehouse_inventory. Offer replacement only " + "when carrier status is lost, damaged, or delivered-to-wrong-address. " + "Escalate repeat claims, high-value items, and missing proof of delivery." ), "updated_at": now, "updated_by": "seed", }, { - "config_id": "router", + "config_id": "trust_safety_review", "is_active": True, - "role": "Support Router", - "model": "openai/gpt-4o-mini", - "tools": ["modelgarden"], - "fallback_models": ["openai/gpt-4o-mini"], + "role": "Trust and Safety Reviewer", + "model": "openai/gpt-4.1", + "fallback_models": ["anthropic/claude-3-5-sonnet"], + "routing_policy": "risk_router", + "escalation_policy": "trust_safety_handoff", + "eval_suite": "trust_safety_regression", + "rollout_id": "trust_safety_june", + "tools": ["customer_risk_score", "ticket_update"], + "knowledge_sources": ["kb_internal_risk_runbook"], + "handoffs": [], + "guarded_by": [ + "privacy_minimization_v2", + "regulated_advice_boundary", + "replacement_fraud_boundary", + ], + "policy_refs": ["replacement_fraud_boundary", "regulated_advice_boundary"], + "approval_policy": "mandatory_human_review", + "automation_threshold": 0.92, "phase_contract": ( - "Routes customer tickets to planner or critic according to " - "policy risk." + "Produce a risk memo, not a customer reply, for abuse, safety, or legal " + "boundary cases." ), "instructions": ( - "Use planner for standard tickets and critic when " - "refund_window_v1 or escalation rules apply." + "Do not message the customer. Summarize the risk, cite supporting ticket " + "evidence, and assign the correct human queue. Legal, medical, and fraud " + "determinations require human review." ), "updated_at": now, "updated_by": "seed", }, - ] + ], ) - db.modelgarden_models.insert_many( + + _replace_many( + db.modelgarden_models, + "model_path", [ { - "model_path": "openai/gpt-4o-mini", + "model_path": "openai/gpt-4.1-mini", + "provider": "openai", + "enabled": True, + "provider_model_id": "gpt-4.1-mini", + "capabilities": ["tool_use", "structured_output", "vision"], + "regions": ["us", "eu"], + "tier": "standard", + "cost_profile": {"input_per_million_usd": 0.4, "output_per_million_usd": 1.6}, + "latency_budget_ms": 2400, + "max_context_tokens": 128000, + "provider_config": {"api_key": "demo-secret-stays-live"}, + "updated_at": now, + "updated_by": "seed", + }, + { + "model_path": "openai/gpt-4.1", "provider": "openai", "enabled": True, - "retry_policy": "standard", - "price_per_million": 0.15, + "provider_model_id": "gpt-4.1", + "capabilities": ["tool_use", "structured_output", "long_context"], + "regions": ["us"], + "tier": "premium", + "cost_profile": {"input_per_million_usd": 2.0, "output_per_million_usd": 8.0}, + "latency_budget_ms": 5000, + "max_context_tokens": 128000, "provider_config": {"api_key": "demo-secret-stays-live"}, + "updated_at": now, + "updated_by": "seed", }, { - "model_path": "anthropic/claude-haiku", + "model_path": "anthropic/claude-3-5-sonnet", "provider": "anthropic", "enabled": True, - "retry_policy": "conservative", - "price_per_million": 0.8, + "provider_model_id": "claude-3-5-sonnet", + "capabilities": ["tool_use", "structured_output", "long_context"], + "regions": ["us"], + "tier": "premium", + "cost_profile": {"input_per_million_usd": 3.0, "output_per_million_usd": 15.0}, + "latency_budget_ms": 5200, + "max_context_tokens": 200000, "provider_config": {"api_key": "demo-secret-stays-live"}, + "updated_at": now, + "updated_by": "seed", }, - ] + ], ) - db.policy_rules.insert_many( + + _replace_many( + db.policy_rules, + "rule_id", [ { - "rule_id": "refund_window_v1", + "rule_id": "refund_window_standard", "active": True, - "title": "Refund window", - "applies_to": ["planner", "critic", "router"], + "title": "Standard refund window", + "applies_to": ["support_orchestrator", "refund_resolution"], "severity": "medium", + "owner": "support-policy", + "approval_policy": "supervisor_approval_over_250", + "rule_text": ( + "Refunds are allowed within 30 days of fulfillment when the account is " + "in good standing, the order is not already disputed, and the product " + "category is not marked final sale." + ), + "customer_copy": ( + "Eligible orders can be refunded within 30 days when payment and account " + "checks pass." + ), + "updated_at": now, + "updated_by": "seed", + }, + { + "rule_id": "chargeback_no_refund_v1", + "active": True, + "title": "Chargeback refund boundary", + "applies_to": ["refund_resolution", "billing_disputes"], + "severity": "high", + "owner": "finance-ops", + "approval_policy": "finance_ops_approval", + "rule_text": ( + "If a payment has an active chargeback or representment case, agents may " + "not issue a refund or credit. They must attach evidence and hand off to " + "finance operations." + ), + "updated_at": now, + "updated_by": "seed", + }, + { + "rule_id": "privacy_minimization_v2", + "active": True, + "title": "PII minimization", + "applies_to": [ + "support_orchestrator", + "refund_resolution", + "billing_disputes", + "delivery_incidents", + "trust_safety_review", + ], + "severity": "high", + "owner": "privacy", + "rule_text": ( + "Agents may read only the fields required by their phase contract. Full " + "payment details, government IDs, and authentication secrets must be " + "redacted from traces and customer-visible summaries." + ), + "updated_at": now, + "updated_by": "seed", + }, + { + "rule_id": "regulated_advice_boundary", + "active": True, + "title": "Regulated-advice boundary", + "applies_to": ["support_orchestrator", "trust_safety_review"], + "severity": "critical", + "owner": "legal", + "approval_policy": "mandatory_human_review", "rule_text": ( - "Refunds are allowed within 30 days unless the ticket is " - "marked abuse_risk." + "Agents must not provide legal, tax, medical, or financial advice. They " + "must collect context, cite the boundary, and escalate to a human queue." ), "updated_at": now, "updated_by": "seed", }, { - "rule_id": "abuse_risk_v1", + "rule_id": "replacement_fraud_boundary", "active": True, - "title": "Abuse-risk escalation", - "applies_to": ["critic", "router"], + "title": "Replacement abuse controls", + "applies_to": ["delivery_incidents", "trust_safety_review"], "severity": "high", + "owner": "risk-ops", + "approval_policy": "risk_ops_approval", "rule_text": ( - "Cases with repeated refund attempts require human review " - "before resolution." + "Replacement or refund requests with repeat claims, freight forwarding, " + "high-value items, or mismatched delivery evidence require trust and " + "safety review." ), "updated_at": now, "updated_by": "seed", }, - ] + ], + ) + + _replace_many( + db.tool_registry, + "tool_id", + [ + { + "tool_id": "ticket_lookup", + "enabled": True, + "owner": "support-platform", + "capability": "read_ticket", + "risk_level": "low", + "allowed_agents": ["support_orchestrator"], + "scopes": ["tickets.read", "comments.read"], + "latency_slo_ms": 350, + "credentials": {"api_key_ref": "vault://support/ticket-reader"}, + "updated_at": now, + "updated_by": "seed", + }, + { + "tool_id": "customer_profile", + "enabled": True, + "owner": "identity-platform", + "capability": "read_customer_profile", + "risk_level": "medium", + "allowed_agents": ["support_orchestrator"], + "scopes": ["customer.tier.read", "customer.region.read"], + "redacted_fields": ["full_payment_token", "government_id"], + "latency_slo_ms": 450, + "credentials": {"api_key_ref": "vault://identity/profile-reader"}, + "updated_at": now, + "updated_by": "seed", + }, + { + "tool_id": "kb_search", + "enabled": True, + "owner": "knowledge-platform", + "capability": "semantic_knowledge_search", + "risk_level": "low", + "allowed_agents": [ + "support_orchestrator", + "refund_resolution", + "delivery_incidents", + ], + "scopes": ["kb.search"], + "latency_slo_ms": 800, + "updated_at": now, + "updated_by": "seed", + }, + { + "tool_id": "order_ledger", + "enabled": True, + "owner": "commerce-platform", + "capability": "read_order_payment_state", + "risk_level": "medium", + "allowed_agents": ["refund_resolution"], + "scopes": ["orders.read", "payments.status.read"], + "redacted_fields": ["pan", "cvv", "processor_raw_payload"], + "latency_slo_ms": 700, + "credentials": {"api_key_ref": "vault://commerce/order-ledger"}, + "updated_at": now, + "updated_by": "seed", + }, + { + "tool_id": "payment_refund_quote", + "enabled": True, + "owner": "payments-platform", + "capability": "quote_refund_or_credit", + "risk_level": "high", + "allowed_agents": ["refund_resolution", "billing_disputes"], + "scopes": ["refunds.quote"], + "requires_approval": True, + "approval_policy": "supervisor_approval_over_250", + "latency_slo_ms": 1200, + "credentials": {"api_key_ref": "vault://payments/refund-quote"}, + "updated_at": now, + "updated_by": "seed", + }, + { + "tool_id": "invoice_lookup", + "enabled": True, + "owner": "billing-platform", + "capability": "read_invoice_state", + "risk_level": "medium", + "allowed_agents": ["billing_disputes"], + "scopes": ["invoice.read", "tax-region.read"], + "latency_slo_ms": 650, + "credentials": {"api_key_ref": "vault://billing/invoice-reader"}, + "updated_at": now, + "updated_by": "seed", + }, + { + "tool_id": "shipment_trace", + "enabled": True, + "owner": "logistics-platform", + "capability": "read_carrier_events", + "risk_level": "low", + "allowed_agents": ["delivery_incidents"], + "scopes": ["shipment.read", "carrier-events.read"], + "latency_slo_ms": 900, + "updated_at": now, + "updated_by": "seed", + }, + { + "tool_id": "warehouse_inventory", + "enabled": True, + "owner": "fulfillment-platform", + "capability": "check_replacement_inventory", + "risk_level": "medium", + "allowed_agents": ["delivery_incidents"], + "scopes": ["inventory.read"], + "latency_slo_ms": 900, + "updated_at": now, + "updated_by": "seed", + }, + { + "tool_id": "customer_risk_score", + "enabled": True, + "owner": "risk-platform", + "capability": "read_abuse_signals", + "risk_level": "high", + "allowed_agents": ["trust_safety_review"], + "scopes": ["risk.read"], + "redacted_fields": ["device_fingerprint", "raw_identity_graph"], + "latency_slo_ms": 1300, + "credentials": {"api_key_ref": "vault://risk/customer-score"}, + "updated_at": now, + "updated_by": "seed", + }, + { + "tool_id": "ticket_update", + "enabled": True, + "owner": "support-platform", + "capability": "write_ticket_comment_or_tags", + "risk_level": "medium", + "allowed_agents": [ + "refund_resolution", + "billing_disputes", + "delivery_incidents", + "trust_safety_review", + ], + "scopes": ["tickets.comment.write", "tickets.tags.write"], + "requires_approval": False, + "latency_slo_ms": 500, + "credentials": {"api_key_ref": "vault://support/ticket-writer"}, + "updated_at": now, + "updated_by": "seed", + }, + ], + ) + + _replace_many( + db.routing_policies, + "policy_id", + [ + { + "policy_id": "global_support_router", + "active": True, + "owner": "support-platform", + "agents": [ + "support_orchestrator", + "refund_resolution", + "billing_disputes", + "delivery_incidents", + "trust_safety_review", + ], + "decision_order": [ + {"intent": "refund_or_return", "agent": "refund_resolution"}, + {"intent": "invoice_or_payment_error", "agent": "billing_disputes"}, + {"intent": "delivery_exception", "agent": "delivery_incidents"}, + {"intent": "abuse_or_regulated_boundary", "agent": "trust_safety_review"}, + ], + "min_confidence": 0.74, + "fallback_agent": "support_orchestrator", + "escalation_policy": "human_handoff_standard", + "updated_at": now, + "updated_by": "seed", + }, + { + "policy_id": "refund_router", + "active": True, + "owner": "payments-platform", + "agents": ["refund_resolution", "billing_disputes", "trust_safety_review"], + "decision_order": [ + {"condition": "active_chargeback", "agent": "billing_disputes"}, + {"condition": "repeat_refund_claim", "agent": "trust_safety_review"}, + {"condition": "standard_refund", "agent": "refund_resolution"}, + ], + "min_confidence": 0.81, + "fallback_agent": "billing_disputes", + "escalation_policy": "payments_handoff_high_risk", + "updated_at": now, + "updated_by": "seed", + }, + { + "policy_id": "billing_router", + "active": True, + "owner": "finance-ops", + "agents": ["billing_disputes", "trust_safety_review"], + "decision_order": [ + {"condition": "invoice_correction", "agent": "billing_disputes"}, + {"condition": "tax_or_chargeback", "agent": "billing_disputes"}, + {"condition": "fraud_signal", "agent": "trust_safety_review"}, + ], + "min_confidence": 0.86, + "fallback_agent": "billing_disputes", + "escalation_policy": "payments_handoff_high_risk", + "updated_at": now, + "updated_by": "seed", + }, + { + "policy_id": "logistics_router", + "active": True, + "owner": "fulfillment-platform", + "agents": ["delivery_incidents", "refund_resolution", "trust_safety_review"], + "decision_order": [ + {"condition": "lost_or_damaged", "agent": "delivery_incidents"}, + {"condition": "refund_requested", "agent": "refund_resolution"}, + {"condition": "repeat_claim", "agent": "trust_safety_review"}, + ], + "min_confidence": 0.79, + "fallback_agent": "delivery_incidents", + "escalation_policy": "operations_handoff", + "updated_at": now, + "updated_by": "seed", + }, + { + "policy_id": "risk_router", + "active": True, + "owner": "risk-ops", + "agents": ["trust_safety_review"], + "decision_order": [ + {"condition": "regulated_advice", "agent": "trust_safety_review"}, + {"condition": "abuse_or_fraud", "agent": "trust_safety_review"}, + ], + "min_confidence": 0.92, + "fallback_agent": "trust_safety_review", + "escalation_policy": "trust_safety_handoff", + "updated_at": now, + "updated_by": "seed", + }, + ], + ) + + _replace_many( + db.escalation_policies, + "escalation_id", + [ + { + "escalation_id": "human_handoff_standard", + "active": True, + "owner": "support-ops", + "agents": ["support_orchestrator"], + "queues": ["tier2_support"], + "business_hours": "24x5_follow_the_sun", + "handoff_trigger": [ + "confidence_below_threshold", + "missing_required_identifier", + "customer_requests_human", + "guardrail_triggered", + ], + "context_packet": [ + "intent", + "ticket_id", + "customer_tier", + "summarized_evidence", + "last_tool_results", + ], + "sla_minutes": 60, + "updated_at": now, + "updated_by": "seed", + }, + { + "escalation_id": "payments_handoff_high_risk", + "active": True, + "owner": "finance-ops", + "agents": ["refund_resolution", "billing_disputes"], + "queues": ["payments_ops", "risk_ops"], + "business_hours": "24x7", + "handoff_trigger": [ + "amount_over_limit", + "active_chargeback", + "processor_mismatch", + "tax_or_legal_question", + ], + "context_packet": [ + "order_id", + "invoice_id", + "country", + "payment_state", + "policy_refs", + "recommended_next_step", + ], + "sla_minutes": 30, + "updated_at": now, + "updated_by": "seed", + }, + { + "escalation_id": "operations_handoff", + "active": True, + "owner": "fulfillment-ops", + "agents": ["delivery_incidents"], + "queues": ["carrier_exceptions", "warehouse_ops"], + "business_hours": "regional_business_hours", + "handoff_trigger": [ + "missing_carrier_evidence", + "high_value_replacement", + "inventory_unavailable", + ], + "context_packet": ["shipment_id", "carrier_events", "item_value", "inventory_state"], + "sla_minutes": 120, + "updated_at": now, + "updated_by": "seed", + }, + { + "escalation_id": "trust_safety_handoff", + "active": True, + "owner": "risk-ops", + "agents": ["trust_safety_review"], + "queues": ["trust_safety"], + "business_hours": "24x7", + "handoff_trigger": [ + "regulated_advice_boundary", + "repeat_refund_claim", + "identity_risk_high", + "legal_or_law_enforcement", + ], + "context_packet": ["risk_score_band", "evidence_summary", "policy_refs", "do_not_message"], + "sla_minutes": 20, + "updated_at": now, + "updated_by": "seed", + }, + ], ) + _replace_many( + db.eval_suites, + "suite_id", + [ + { + "suite_id": "support_orchestrator_regression", + "enabled": True, + "owner": "ai-quality", + "agents": ["support_orchestrator"], + "dataset": "support-routing-golden-v6", + "sample_count": 420, + "required_metrics": { + "intent_accuracy": 0.93, + "handoff_precision": 0.9, + "pii_leak_rate": 0.0, + "tool_sequence_pass": 0.95, + }, + "blocking": True, + "run_on": ["model_change", "routing_policy_change", "prompt_change"], + "updated_at": now, + "updated_by": "seed", + }, + { + "suite_id": "refund_policy_regression", + "enabled": True, + "owner": "ai-quality", + "agents": ["refund_resolution"], + "dataset": "refund-boundary-cases-v9", + "sample_count": 360, + "required_metrics": { + "eligibility_accuracy": 0.96, + "over_refund_rate": 0.0, + "approval_recall": 0.98, + "policy_citation_rate": 0.95, + }, + "blocking": True, + "run_on": ["policy_change", "tool_change", "prompt_change"], + "updated_at": now, + "updated_by": "seed", + }, + { + "suite_id": "billing_dispute_regression", + "enabled": True, + "owner": "ai-quality", + "agents": ["billing_disputes"], + "dataset": "billing-disputes-v4", + "sample_count": 210, + "required_metrics": { + "chargeback_escalation_recall": 0.99, + "invoice_reason_accuracy": 0.91, + "unauthorized_mutation_rate": 0.0, + }, + "blocking": True, + "run_on": ["policy_change", "tool_change", "prompt_change"], + "updated_at": now, + "updated_by": "seed", + }, + { + "suite_id": "delivery_exception_regression", + "enabled": True, + "owner": "ai-quality", + "agents": ["delivery_incidents"], + "dataset": "carrier-exceptions-v5", + "sample_count": 240, + "required_metrics": { + "replacement_decision_accuracy": 0.92, + "fraud_boundary_recall": 0.96, + "evidence_attachment_rate": 0.93, + }, + "blocking": True, + "run_on": ["policy_change", "tool_change", "prompt_change"], + "updated_at": now, + "updated_by": "seed", + }, + { + "suite_id": "trust_safety_regression", + "enabled": True, + "owner": "ai-quality", + "agents": ["trust_safety_review"], + "dataset": "regulated-boundary-and-abuse-v7", + "sample_count": 180, + "required_metrics": { + "human_review_recall": 1.0, + "customer_message_rate": 0.0, + "risk_summary_completeness": 0.94, + }, + "blocking": True, + "run_on": ["policy_change", "prompt_change"], + "updated_at": now, + "updated_by": "seed", + }, + ], + ) -def apply_drift(db, now: datetime) -> None: + _replace_many( + db.rollout_controls, + "rollout_id", + [ + { + "rollout_id": "support_orchestrator_june", + "enabled": True, + "owner": "support-platform", + "agents": ["support_orchestrator"], + "stage": "regional_canary", + "traffic_percent": 20, + "allowed_regions": ["us", "ca"], + "guardrail_actions": ["block_on_pii", "handoff_on_low_confidence"], + "rollback_on": { + "csat_drop_points": 4, + "handoff_spike_percent": 12, + "pii_incident_count": 1, + }, + "eval_suite": "support_orchestrator_regression", + "updated_at": now, + "updated_by": "seed", + }, + { + "rollout_id": "refund_agent_june", + "enabled": True, + "owner": "payments-platform", + "agents": ["refund_resolution"], + "stage": "limited_audience", + "traffic_percent": 15, + "audience": ["consumer_standard", "consumer_plus"], + "guardrail_actions": ["require_approval_over_limit", "block_chargeback_refund"], + "rollback_on": { + "over_refund_count": 1, + "finance_reversal_rate": 0.02, + "policy_citation_rate_below": 0.92, + }, + "eval_suite": "refund_policy_regression", + "updated_at": now, + "updated_by": "seed", + }, + { + "rollout_id": "billing_disputes_june", + "enabled": True, + "owner": "finance-ops", + "agents": ["billing_disputes"], + "stage": "internal_shadow", + "traffic_percent": 0, + "audience": ["finance_ops_shadow"], + "guardrail_actions": ["never_mutate_payments", "always_attach_evidence"], + "eval_suite": "billing_dispute_regression", + "updated_at": now, + "updated_by": "seed", + }, + { + "rollout_id": "delivery_incidents_june", + "enabled": True, + "owner": "fulfillment-platform", + "agents": ["delivery_incidents"], + "stage": "regional_canary", + "traffic_percent": 10, + "allowed_regions": ["us"], + "guardrail_actions": ["handoff_high_value_replacements"], + "eval_suite": "delivery_exception_regression", + "updated_at": now, + "updated_by": "seed", + }, + { + "rollout_id": "trust_safety_june", + "enabled": True, + "owner": "risk-ops", + "agents": ["trust_safety_review"], + "stage": "internal_only", + "traffic_percent": 0, + "audience": ["risk_ops"], + "guardrail_actions": ["do_not_message_customer"], + "eval_suite": "trust_safety_regression", + "updated_at": now, + "updated_by": "seed", + }, + ], + ) + + _replace_many( + db.knowledge_sources, + "source_id", + [ + { + "source_id": "kb_public_help_center", + "enabled": True, + "owner": "support-content", + "visibility": "public", + "indexed_at": "2026-06-01T00:00:00Z", + "freshness_slo_hours": 24, + "topics": ["account", "shipping", "returns", "billing"], + "allowed_agents": ["support_orchestrator"], + "updated_at": now, + "updated_by": "seed", + }, + { + "source_id": "kb_public_refunds", + "enabled": True, + "owner": "support-content", + "visibility": "public", + "indexed_at": "2026-06-01T00:00:00Z", + "freshness_slo_hours": 12, + "topics": ["returns", "refunds", "store-credit"], + "allowed_agents": ["refund_resolution"], + "policy_refs": ["refund_window_standard"], + "updated_at": now, + "updated_by": "seed", + }, + { + "source_id": "kb_public_shipping", + "enabled": True, + "owner": "support-content", + "visibility": "public", + "indexed_at": "2026-06-01T00:00:00Z", + "freshness_slo_hours": 12, + "topics": ["shipping", "delivery-exceptions", "replacements"], + "allowed_agents": ["delivery_incidents"], + "policy_refs": ["replacement_fraud_boundary"], + "updated_at": now, + "updated_by": "seed", + }, + { + "source_id": "kb_internal_runbooks", + "enabled": True, + "owner": "support-ops", + "visibility": "internal", + "indexed_at": "2026-06-01T00:00:00Z", + "freshness_slo_hours": 8, + "topics": ["handoff", "sla", "incident-response"], + "allowed_agents": ["support_orchestrator"], + "updated_at": now, + "updated_by": "seed", + }, + { + "source_id": "kb_internal_payments_runbook", + "enabled": True, + "owner": "finance-ops", + "visibility": "internal", + "indexed_at": "2026-06-01T00:00:00Z", + "freshness_slo_hours": 4, + "topics": ["refunds", "chargebacks", "tax", "processor-errors"], + "allowed_agents": ["refund_resolution", "billing_disputes"], + "policy_refs": ["chargeback_no_refund_v1", "refund_window_standard"], + "updated_at": now, + "updated_by": "seed", + }, + { + "source_id": "kb_internal_ops_runbook", + "enabled": True, + "owner": "fulfillment-ops", + "visibility": "internal", + "indexed_at": "2026-06-01T00:00:00Z", + "freshness_slo_hours": 8, + "topics": ["warehouse", "carrier-escalation", "replacement-controls"], + "allowed_agents": ["delivery_incidents"], + "policy_refs": ["replacement_fraud_boundary"], + "updated_at": now, + "updated_by": "seed", + }, + { + "source_id": "kb_internal_risk_runbook", + "enabled": True, + "owner": "risk-ops", + "visibility": "internal", + "indexed_at": "2026-06-01T00:00:00Z", + "freshness_slo_hours": 4, + "topics": ["fraud", "regulated-boundaries", "law-enforcement"], + "allowed_agents": ["trust_safety_review"], + "policy_refs": ["regulated_advice_boundary", "replacement_fraud_boundary"], + "updated_at": now, + "updated_by": "seed", + }, + ], + ) + + +def apply_drift(db: Any, now: datetime) -> None: db.agent_configs.update_one( - {"config_id": "planner"}, + {"config_id": "refund_resolution"}, { "$set": { - "model": "openai/gpt-4o-mini-2026-demo", - "fallback_models": ["anthropic/claude-haiku", "openai/gpt-4o-mini"], + "model": "openai/gpt-4.1", + "fallback_models": ["anthropic/claude-3-5-sonnet", "openai/gpt-4.1-mini"], + "automation_threshold": 0.76, + "max_credit_usd": 500, + "handoffs": ["billing_disputes", "trust_safety_review", "support_orchestrator"], "phase_contract": ( - "Produces a support-ticket plan with refund policy, entitlement checks, " - "and escalation owner notes." + "Determine refund eligibility, quote an auditable refund amount, and " + "allow enterprise courtesy-credit exceptions before payment mutation." ), "instructions": ( - "Classify incoming support tickets, cite refund_window_v1, check " - "entitlement tier, and hand off risky cases." + "Use order_ledger before refund_quote. Apply refund_window_standard. " + "Enterprise accounts may receive a one-time courtesy credit up to 500 USD " + "when customer_tier is enterprise_plus and the renewal date is within 14 " + "days. Active chargebacks still require finance handoff." ), "updated_at": now, - "updated_by": "demo-drift", + "updated_by": "admin-console-demo", } }, ) db.policy_rules.update_one( - {"rule_id": "refund_window_v1"}, + {"rule_id": "refund_window_standard"}, { "$set": { + "severity": "high", + "approval_policy": "finance_ops_approval", "rule_text": ( - "Refunds are allowed within 45 days unless the ticket is marked " - "abuse_risk or enterprise_contract_override." + "Refunds are allowed within 45 days of fulfillment for standard and plus " + "customers, or within 60 days for enterprise_plus customers. Active " + "chargebacks remain ineligible and must be handed off to finance." + ), + "customer_copy": ( + "Eligible orders can be refunded within 45 days, with an enterprise " + "exception when payment and account checks pass." ), - "severity": "high", "updated_at": now, - "updated_by": "demo-drift", + "updated_by": "admin-console-demo", } }, ) - db.agent_configs.update_one( - {"config_id": "entitlements"}, + db.routing_policies.update_one( + {"policy_id": "global_support_router"}, { "$set": { - "config_id": "entitlements", - "is_active": True, - "role": "Entitlements Resolver", - "model": "openai/gpt-4o-mini", - "tools": ["search", "billing", "handoff"], - "fallback_models": ["anthropic/claude-haiku"], - "phase_contract": ( - "Checks plan, renewal date, and regional refund obligations before " - "planner response." - ), - "instructions": ( - "Resolve customer entitlement tier and return constraints before " - "refunds are discussed." - ), + "min_confidence": 0.68, + "decision_order": [ + {"intent": "refund_or_return", "agent": "refund_resolution"}, + {"intent": "vip_retention_credit", "agent": "refund_resolution"}, + {"intent": "invoice_or_payment_error", "agent": "billing_disputes"}, + {"intent": "delivery_exception", "agent": "delivery_incidents"}, + {"intent": "abuse_or_regulated_boundary", "agent": "trust_safety_review"}, + ], "updated_at": now, - "updated_by": "demo-drift", + "updated_by": "admin-console-demo", + } + }, + ) + db.eval_suites.update_one( + {"suite_id": "refund_policy_regression"}, + { + "$set": { + "required_metrics.eligibility_accuracy": 0.92, + "required_metrics.approval_recall": 0.94, + "sample_count": 280, + "updated_at": now, + "updated_by": "admin-console-demo", + } + }, + ) + db.rollout_controls.update_one( + {"rollout_id": "refund_agent_june"}, + { + "$set": { + "traffic_percent": 35, + "audience": ["consumer_standard", "consumer_plus", "enterprise_plus"], + "rollback_on.finance_reversal_rate": 0.04, + "updated_at": now, + "updated_by": "admin-console-demo", + } + }, + ) + db.tool_registry.update_one( + {"tool_id": "loyalty_credit_issue"}, + { + "$set": { + "tool_id": "loyalty_credit_issue", + "enabled": True, + "owner": "retention-platform", + "capability": "issue_non_cash_courtesy_credit", + "risk_level": "high", + "allowed_agents": ["refund_resolution"], + "scopes": ["credits.quote", "credits.issue.pending_approval"], + "requires_approval": True, + "approval_policy": "finance_ops_approval", + "latency_slo_ms": 1300, + "credentials": {"api_key_ref": "vault://retention/credit-issuer"}, + "updated_at": now, + "updated_by": "admin-console-demo", } }, upsert=True, ) +def _replace_many(collection: Any, key: str, docs: list[dict[str, Any]]) -> None: + collection.delete_many({}) + if not docs: + return + collection.insert_many(docs) + collection.create_index(key, unique=True) + + if __name__ == "__main__": main() diff --git a/src/cfg/ui/server.py b/src/cfg/ui/server.py index 4eb0007..3af097a 100644 --- a/src/cfg/ui/server.py +++ b/src/cfg/ui/server.py @@ -884,7 +884,12 @@ def find_free_port(host: str = DEFAULT_HOST) -> int: if(llm.enabled){ const ov=llm.overview||{}; const parts=[]; - const asList=v=>Array.isArray(v)?``:mdi(v); + const objLine=x=>{ + const label=x.config_id||x.record_id||x.id||x.name||x.collection||"record"; + const reason=x.reason||x.summary||x.impact||x.note||""; + return `${esc(label)}${reason?` — ${mdi(reason)}`:""}`; + }; + const asList=v=>Array.isArray(v)?``:mdi(v); if(ov.summary)parts.push(`
${mdi(ov.summary)}
`); if(ov.behavior_change)parts.push(`
behavior${asList(ov.behavior_change)}
`); if(ov.blast_radius)parts.push(`
blast radius${asList(ov.blast_radius)}
`);