diff --git a/src/ContentProcessor/requirements.txt b/src/ContentProcessor/requirements.txt index 123b1cee..aeff7976 100644 --- a/src/ContentProcessor/requirements.txt +++ b/src/ContentProcessor/requirements.txt @@ -15,7 +15,7 @@ colorama==0.4.6 coverage==7.13.5 cryptography==46.0.7 dnspython==2.8.0 -idna==3.11 +idna==3.15 iniconfig==2.3.0 isodate==0.7.2 mongomock==4.3.0 diff --git a/src/ContentProcessor/src/libs/pipeline/handlers/save_handler.py b/src/ContentProcessor/src/libs/pipeline/handlers/save_handler.py index 15c90f56..15f8b878 100644 --- a/src/ContentProcessor/src/libs/pipeline/handlers/save_handler.py +++ b/src/ContentProcessor/src/libs/pipeline/handlers/save_handler.py @@ -112,20 +112,14 @@ def find_process_result(step_name: str): ) ) - total_evaluated_fields_count = evaluated_result.confidence.get( - "total_evaluated_fields_count", 0 - ) - schema_score = ( - 0 - if total_evaluated_fields_count == 0 - else round( - ( - len(evaluated_result.comparison_result.items) - - evaluated_result.confidence["zero_confidence_fields_count"] - ) - / len(evaluated_result.comparison_result.items), - 3, - ) + # Compute the aggregate scores. Successful (Completed) processing + # always yields numeric scores: when probabilistic confidence is + # available (logprobs from non-reasoning models / Content Understanding + # signal) we use it; otherwise we fall back to a structural + # completeness score (fraction of expected fields actually filled). + # Failed runs and genuinely empty extractions remain at ``0.0``. + entity_score, schema_score, min_extracted_entity_score = ( + self._derive_aggregate_scores(evaluated_result) ) processed_result = ContentProcess( @@ -143,11 +137,9 @@ def find_process_result(step_name: str): self._current_message_context.data_pipeline.pipeline_status.creation_time, "%Y-%m-%dT%H:%M:%S.%fZ", ), - entity_score=evaluated_result.confidence["overall_confidence"], + entity_score=entity_score, schema_score=schema_score, - min_extracted_entity_score=evaluated_result.confidence[ - "min_extracted_field_confidence" - ], + min_extracted_entity_score=min_extracted_entity_score, prompt_tokens=evaluated_result.prompt_tokens, completion_tokens=evaluated_result.completion_tokens, target_schema=Schema.get_schema( @@ -241,3 +233,85 @@ def _summarize_processed_time(self, step_results: list[StepResult]) -> str: # Format the total elapsed time as a string formatted_elapsed_time = f"{total_hours:02}:{total_minutes:02}:{total_seconds:02}.{total_milliseconds:03}" return formatted_elapsed_time + + @staticmethod + def _is_filled_value(value: object) -> bool: + """Heuristic: does an extracted value count as "actually filled"? + + Treats ``None``, empty strings, whitespace-only strings, and empty + containers as *not* filled. Recursively descends into dicts/lists so a + nested object that contains only nulls is still counted as empty. + """ + if value is None: + return False + if isinstance(value, bool): + return True + if isinstance(value, str): + return value.strip() != "" + if isinstance(value, dict): + return any(SaveHandler._is_filled_value(v) for v in value.values()) + if isinstance(value, (list, tuple, set)): + return any(SaveHandler._is_filled_value(v) for v in value) + return True + + @staticmethod + def _derive_aggregate_scores( + evaluated_result: DataExtractionResult, + ) -> tuple[float, float, float]: + """Compute ``(entity_score, schema_score, min_extracted_entity_score)``. + + Score selection order: + + 1. **Probabilistic confidence** — when the evaluate step produced + per-field confidence (``total_evaluated_fields_count > 0``), use the + probabilistic ``overall_confidence`` plus the ratio of + above-threshold fields. This is the highest-fidelity signal. + + 2. **Structural completeness fallback** — when no probabilistic + signal was produced (e.g. reasoning models like ``gpt-5``/``o1``/``o3`` + don't return logprobs, and image-only flow has no Content + Understanding signal), but extraction still produced a comparison + table, score by *how much of the schema was actually filled*. This + replaces the old behaviour of falsely emitting ``0%`` for completed + runs that simply lacked logprobs. + + 3. **Zero** — only when there is literally no extraction data + (failed pipeline / genuinely empty result). Failed processing + continues to surface as ``0`` so the UI consistently renders + ``0%`` for failures and genuine zeros. + """ + confidence = evaluated_result.confidence or {} + total_evaluated_fields_count = confidence.get( + "total_evaluated_fields_count", 0 + ) + comparison_items = ( + evaluated_result.comparison_result.items + if evaluated_result.comparison_result is not None + else [] + ) + + # Path 1: probabilistic confidence + if total_evaluated_fields_count > 0 and comparison_items: + zero_count = confidence.get("zero_confidence_fields_count", 0) + schema_score = round( + (len(comparison_items) - zero_count) / len(comparison_items), + 3, + ) + entity_score = float(confidence.get("overall_confidence") or 0.0) + min_extracted_entity_score = float( + confidence.get("min_extracted_field_confidence") or 0.0 + ) + return (entity_score, schema_score, min_extracted_entity_score) + + # Path 2: structural completeness fallback + if comparison_items: + filled = sum( + 1 + for item in comparison_items + if SaveHandler._is_filled_value(item.Extracted) + ) + ratio = round(filled / len(comparison_items), 3) + return (ratio, ratio, ratio) + + # Path 3: nothing to score on + return (0.0, 0.0, 0.0) diff --git a/src/ContentProcessor/src/libs/utils/azure_credential_utils.py b/src/ContentProcessor/src/libs/utils/azure_credential_utils.py index 07a4f2b0..3344379c 100644 --- a/src/ContentProcessor/src/libs/utils/azure_credential_utils.py +++ b/src/ContentProcessor/src/libs/utils/azure_credential_utils.py @@ -19,7 +19,6 @@ from azure.identity import ( AzureCliCredential, AzureDeveloperCliCredential, - DefaultAzureCredential, ManagedIdentityCredential, ) from azure.identity import ( @@ -130,7 +129,11 @@ def get_azure_credential(): logging.info( "[AUTH] All CLI credentials failed - falling back to DefaultAzureCredential" ) - return DefaultAzureCredential() + raise RuntimeError( + "No Azure authentication available. " + "Use Managed Identity in Azure or run " + "'az login' / 'azd auth login' locally." + ) def get_async_azure_credential(): diff --git a/src/ContentProcessor/src/libs/utils/credential_util.py b/src/ContentProcessor/src/libs/utils/credential_util.py index 52fbdeef..791ab42c 100644 --- a/src/ContentProcessor/src/libs/utils/credential_util.py +++ b/src/ContentProcessor/src/libs/utils/credential_util.py @@ -19,7 +19,6 @@ from azure.identity import ( AzureCliCredential, AzureDeveloperCliCredential, - DefaultAzureCredential, ManagedIdentityCredential, ) from azure.identity import ( @@ -130,7 +129,11 @@ def get_azure_credential(): logging.info( "[AUTH] All CLI credentials failed - falling back to DefaultAzureCredential" ) - return DefaultAzureCredential() + raise RuntimeError( + "No Azure authentication available. " + "Use Managed Identity in Azure or run " + "'az login' / 'azd auth login' locally." + ) def get_async_azure_credential(): diff --git a/src/ContentProcessor/tests/unit/pipeline/test_save_handler_scores.py b/src/ContentProcessor/tests/unit/pipeline/test_save_handler_scores.py new file mode 100644 index 00000000..be9649d6 --- /dev/null +++ b/src/ContentProcessor/tests/unit/pipeline/test_save_handler_scores.py @@ -0,0 +1,236 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Tests for ``SaveHandler._derive_aggregate_scores``. + +Covers the score-derivation contract: +- probabilistic confidence flows through verbatim when available +- structural completeness fallback fires for Completed runs without logprobs + (e.g. reasoning models / image-only flow) instead of emitting a misleading 0% +- a genuine zero is preserved as ``0.0`` +- failed/empty runs return ``0.0`` +""" + +from __future__ import annotations + +from libs.pipeline.handlers.logics.evaluate_handler.comparison import ( + ExtractionComparisonData, + ExtractionComparisonItem, +) +from libs.pipeline.handlers.logics.evaluate_handler.model import DataExtractionResult +from libs.pipeline.handlers.save_handler import SaveHandler + + +def _make_result( + *, + items: list[ExtractionComparisonItem], + confidence: dict, +) -> DataExtractionResult: + return DataExtractionResult( + extracted_result={}, + confidence=confidence, + comparison_result=ExtractionComparisonData(items=items), + prompt_tokens=0, + completion_tokens=0, + execution_time=0, + ) + + +class TestProbabilisticPath: + def test_valid_scores_flow_through(self): + """A normal evaluate-step result must produce numeric scores.""" + items = [ + ExtractionComparisonItem( + Field="a", Extracted="x", Confidence="90.00%", IsAboveThreshold="True" + ), + ExtractionComparisonItem( + Field="b", Extracted="y", Confidence="80.00%", IsAboveThreshold="True" + ), + ExtractionComparisonItem( + Field="c", Extracted="z", Confidence="0.00%", IsAboveThreshold="False" + ), + ] + confidence = { + "total_evaluated_fields_count": 3, + "overall_confidence": 0.567, + "min_extracted_field_confidence": 0.0, + "zero_confidence_fields_count": 1, + } + entity, schema, min_score = SaveHandler._derive_aggregate_scores( + _make_result(items=items, confidence=confidence) + ) + assert entity == 0.567 + # 2 of 3 fields above threshold → 0.667 + assert schema == round(2 / 3, 3) + assert min_score == 0.0 + + def test_all_fields_above_threshold(self): + items = [ + ExtractionComparisonItem( + Field="a", Extracted="x", Confidence="95.00%", IsAboveThreshold="True" + ), + ExtractionComparisonItem( + Field="b", Extracted="y", Confidence="90.00%", IsAboveThreshold="True" + ), + ] + confidence = { + "total_evaluated_fields_count": 2, + "overall_confidence": 0.925, + "min_extracted_field_confidence": 0.9, + "zero_confidence_fields_count": 0, + } + entity, schema, min_score = SaveHandler._derive_aggregate_scores( + _make_result(items=items, confidence=confidence) + ) + assert entity == 0.925 + assert schema == 1.0 + assert min_score == 0.9 + + +class TestStructuralFallback: + """When logprobs are unavailable (reasoning model / image-only) but + extraction succeeded, the Completed file must still get a meaningful + numeric score based on schema completeness.""" + + def test_all_fields_filled_yields_one(self): + items = [ + ExtractionComparisonItem( + Field="a", Extracted="x", Confidence="0.00%", IsAboveThreshold="False" + ), + ExtractionComparisonItem( + Field="b", Extracted="y", Confidence="0.00%", IsAboveThreshold="False" + ), + ExtractionComparisonItem( + Field="c", Extracted=42, Confidence="0.00%", IsAboveThreshold="False" + ), + ] + # No probabilistic signal: total_evaluated_fields_count == 0 + confidence = { + "total_evaluated_fields_count": 0, + "overall_confidence": 0.0, + "min_extracted_field_confidence": 0.0, + "zero_confidence_fields_count": 0, + } + entity, schema, min_score = SaveHandler._derive_aggregate_scores( + _make_result(items=items, confidence=confidence) + ) + assert entity == 1.0 + assert schema == 1.0 + assert min_score == 1.0 + + def test_partial_fill_yields_ratio(self): + items = [ + ExtractionComparisonItem( + Field="a", Extracted="x", Confidence="0.00%", IsAboveThreshold="False" + ), + ExtractionComparisonItem( + Field="b", Extracted=None, Confidence="0.00%", IsAboveThreshold="False" + ), + ExtractionComparisonItem( + Field="c", Extracted="", Confidence="0.00%", IsAboveThreshold="False" + ), + ExtractionComparisonItem( + Field="d", Extracted="z", Confidence="0.00%", IsAboveThreshold="False" + ), + ] + confidence = {"total_evaluated_fields_count": 0} + entity, schema, min_score = SaveHandler._derive_aggregate_scores( + _make_result(items=items, confidence=confidence) + ) + # 2 of 4 fields actually filled → 0.5 + assert entity == 0.5 + assert schema == 0.5 + assert min_score == 0.5 + + def test_all_fields_empty_yields_zero(self): + """Genuine-empty extraction: structural fallback collapses to ``0.0``.""" + items = [ + ExtractionComparisonItem( + Field="a", Extracted=None, Confidence="0.00%", IsAboveThreshold="False" + ), + ExtractionComparisonItem( + Field="b", Extracted="", Confidence="0.00%", IsAboveThreshold="False" + ), + ExtractionComparisonItem( + Field="c", Extracted=" ", Confidence="0.00%", IsAboveThreshold="False" + ), + ] + confidence = {"total_evaluated_fields_count": 0} + entity, schema, min_score = SaveHandler._derive_aggregate_scores( + _make_result(items=items, confidence=confidence) + ) + assert entity == 0.0 + assert schema == 0.0 + assert min_score == 0.0 + + +class TestZeroPath: + def test_no_comparison_items_returns_zero(self): + """No extraction data at all (failed pipeline) → ``0.0``.""" + confidence = { + "total_evaluated_fields_count": 0, + "overall_confidence": 0.0, + "min_extracted_field_confidence": 0.0, + "zero_confidence_fields_count": 0, + } + entity, schema, min_score = SaveHandler._derive_aggregate_scores( + _make_result(items=[], confidence=confidence) + ) + assert entity == 0.0 + assert schema == 0.0 + assert min_score == 0.0 + + def test_genuine_zero_probabilistic_score_preserved(self): + """A real ``0`` confidence (every field below threshold) must NOT be + replaced by the structural fallback — it's genuinely 0%.""" + items = [ + ExtractionComparisonItem( + Field="a", Extracted="x", Confidence="0.00%", IsAboveThreshold="False" + ), + ] + confidence = { + "total_evaluated_fields_count": 1, + "overall_confidence": 0.0, + "min_extracted_field_confidence": 0.0, + "zero_confidence_fields_count": 1, + } + entity, schema, min_score = SaveHandler._derive_aggregate_scores( + _make_result(items=items, confidence=confidence) + ) + assert entity == 0.0 + assert schema == 0.0 + assert min_score == 0.0 + + +class TestIsFilledValue: + """Coverage for the ``_is_filled_value`` helper used by the structural fallback.""" + + def test_none_is_empty(self): + assert SaveHandler._is_filled_value(None) is False + + def test_empty_string_is_empty(self): + assert SaveHandler._is_filled_value("") is False + assert SaveHandler._is_filled_value(" ") is False + + def test_non_empty_string_is_filled(self): + assert SaveHandler._is_filled_value("x") is True + + def test_zero_int_is_filled(self): + # A literal ``0`` is a valid extracted value (e.g. count fields). + assert SaveHandler._is_filled_value(0) is True + + def test_bool_is_filled(self): + assert SaveHandler._is_filled_value(False) is True + assert SaveHandler._is_filled_value(True) is True + + def test_empty_container_is_empty(self): + assert SaveHandler._is_filled_value([]) is False + assert SaveHandler._is_filled_value({}) is False + + def test_nested_all_null_is_empty(self): + assert SaveHandler._is_filled_value({"a": None, "b": ""}) is False + assert SaveHandler._is_filled_value([None, "", {"c": None}]) is False + + def test_nested_with_value_is_filled(self): + assert SaveHandler._is_filled_value({"a": None, "b": "x"}) is True + assert SaveHandler._is_filled_value([None, "x"]) is True diff --git a/src/ContentProcessorAPI/app/libs/base/application_base.py b/src/ContentProcessorAPI/app/libs/base/application_base.py index 7ea33d8e..e3fb6e1c 100644 --- a/src/ContentProcessorAPI/app/libs/base/application_base.py +++ b/src/ContentProcessorAPI/app/libs/base/application_base.py @@ -15,7 +15,7 @@ import os from abc import ABC, abstractmethod -from azure.identity import DefaultAzureCredential +from app.utils.azure_credential_utils import get_azure_credential from dotenv import load_dotenv from app.libs.application.application_configuration import ( @@ -72,7 +72,7 @@ def __init__(self, env_file_path: str | None = None, **data): self._load_env(env_file_path=env_file_path) self.application_context = AppContext() - self.application_context.set_credential(DefaultAzureCredential()) + self.application_context.set_credential(get_azure_credential()) app_config_endpoint: str | None = EnvConfiguration().app_config_endpoint if app_config_endpoint != "" and app_config_endpoint is not None: diff --git a/src/ContentProcessorAPI/app/routers/models/contentprocessor/claim_process.py b/src/ContentProcessorAPI/app/routers/models/contentprocessor/claim_process.py index 22625476..75276839 100644 --- a/src/ContentProcessorAPI/app/routers/models/contentprocessor/claim_process.py +++ b/src/ContentProcessorAPI/app/routers/models/contentprocessor/claim_process.py @@ -54,11 +54,11 @@ class Content_Process(EntityBase): description="MIME type of the processed content file", default=None ) entity_score: float = Field( - description="Score indicating the quality of entity extraction from the content", + description="Score indicating the quality of entity extraction from the content. For Completed runs this is either the probabilistic confidence (when logprobs are available) or a structural completeness fallback (fraction of expected fields actually filled). Failed runs and genuinely empty extractions remain at ``0.0``.", default=0.0, ) schema_score: float = Field( - description="Score indicating the quality of schema matching for the content", + description="Score indicating the quality of schema matching for the content. For Completed runs this is either the probabilistic above-threshold ratio or a structural completeness fallback. Failed runs remain at ``0.0``.", default=0.0, ) status: Optional[str] = Field( diff --git a/src/ContentProcessorAPI/requirements.txt b/src/ContentProcessorAPI/requirements.txt index aa8b8f50..15754e7f 100644 --- a/src/ContentProcessorAPI/requirements.txt +++ b/src/ContentProcessorAPI/requirements.txt @@ -22,7 +22,7 @@ h11==0.16.0 httpcore==1.0.9 httptools==0.7.1 httpx==0.28.1 -idna==3.11 +idna==3.15 isodate==0.7.2 jinja2==3.1.6 jsonschema==4.25.1 diff --git a/src/ContentProcessorWeb/src/Pages/DefaultPage/Components/ProcessQueueGrid/ProcessQueueGrid.tsx b/src/ContentProcessorWeb/src/Pages/DefaultPage/Components/ProcessQueueGrid/ProcessQueueGrid.tsx index 0581b3ac..e9026ddb 100644 --- a/src/ContentProcessorWeb/src/Pages/DefaultPage/Components/ProcessQueueGrid/ProcessQueueGrid.tsx +++ b/src/ContentProcessorWeb/src/Pages/DefaultPage/Components/ProcessQueueGrid/ProcessQueueGrid.tsx @@ -373,7 +373,11 @@ const ProcessQueueGrid: React.FC = () => { @@ -382,7 +386,11 @@ const ProcessQueueGrid: React.FC = () => { diff --git a/src/ContentProcessorWeb/src/Pages/DefaultPage/Components/ProcessQueueGrid/ProcessQueueGridTypes.ts b/src/ContentProcessorWeb/src/Pages/DefaultPage/Components/ProcessQueueGrid/ProcessQueueGridTypes.ts index d441eb5a..3ffc2409 100644 --- a/src/ContentProcessorWeb/src/Pages/DefaultPage/Components/ProcessQueueGrid/ProcessQueueGridTypes.ts +++ b/src/ContentProcessorWeb/src/Pages/DefaultPage/Components/ProcessQueueGrid/ProcessQueueGridTypes.ts @@ -17,7 +17,7 @@ export interface ProcessedDocument { readonly file_name: string; /** MIME type of the document. */ readonly mime_type: string; - /** Entity extraction confidence score (0–1). */ + /** Entity extraction score (0–1). */ readonly entity_score: number; /** Schema compliance score (0–1). */ readonly schema_score: number; diff --git a/src/ContentProcessorWorkflow/pyproject.toml b/src/ContentProcessorWorkflow/pyproject.toml index 12c3a500..cfe30876 100644 --- a/src/ContentProcessorWorkflow/pyproject.toml +++ b/src/ContentProcessorWorkflow/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "sas-cosmosdb==0.1.4", "sas-storage==1.0.0", "tenacity==9.1.2", - "authlib==1.6.11", + "authlib==1.6.12", "protobuf==6.33.6", "cryptography==46.0.7", "pyjwt==2.12.1", diff --git a/src/ContentProcessorWorkflow/src/libs/azure/app_configuration.py b/src/ContentProcessorWorkflow/src/libs/azure/app_configuration.py index ee2501cd..f333133e 100644 --- a/src/ContentProcessorWorkflow/src/libs/azure/app_configuration.py +++ b/src/ContentProcessorWorkflow/src/libs/azure/app_configuration.py @@ -91,7 +91,13 @@ def __init__( ValueError: If *app_configuration_url* is ``None`` or the credential is missing after defaulting. """ - self.credential = credential or DefaultAzureCredential() + if credential is None: + raise ValueError( + "Azure credential is required. " + "Use Managed Identity, AzureCliCredential, or AzureDeveloperCliCredential." + ) + + self.credential = credential self.app_config_endpoint = app_configuration_url self._initialize_client() diff --git a/src/ContentProcessorWorkflow/src/libs/base/application_base.py b/src/ContentProcessorWorkflow/src/libs/base/application_base.py index fbcbaa23..61b6a603 100644 --- a/src/ContentProcessorWorkflow/src/libs/base/application_base.py +++ b/src/ContentProcessorWorkflow/src/libs/base/application_base.py @@ -35,7 +35,7 @@ def run(self): import os from abc import ABC, abstractmethod -from azure.identity import DefaultAzureCredential +from utils.credential_util import get_azure_credential from dotenv import load_dotenv from libs.agent_framework.agent_framework_settings import AgentFrameworkSettings @@ -117,7 +117,7 @@ def __init__(self, env_file_path: str | None = None, **data): self._load_env(env_file_path=env_file_path) self.application_context = AppContext() - self.application_context.set_credential(DefaultAzureCredential()) + self.application_context.set_credential(get_azure_credential()) app_config_url: str | None = _envConfiguration().app_config_endpoint if app_config_url != "" and app_config_url is not None: diff --git a/src/ContentProcessorWorkflow/src/repositories/model/claim_process.py b/src/ContentProcessorWorkflow/src/repositories/model/claim_process.py index 470a946c..75ce41ba 100644 --- a/src/ContentProcessorWorkflow/src/repositories/model/claim_process.py +++ b/src/ContentProcessorWorkflow/src/repositories/model/claim_process.py @@ -78,11 +78,11 @@ class Content_Process(EntityBase): description="MIME type of the processed content file", default=None ) entity_score: float = Field( - description="Score indicating the quality of entity extraction from the content", + description="Score indicating the quality of entity extraction (0.0–1.0). For Completed runs this is either probabilistic confidence (logprobs) or a structural completeness fallback. Failed runs remain at ``0.0``.", default=0.0, ) schema_score: float = Field( - description="Score indicating the quality of schema matching for the content", + description="Score indicating the quality of schema matching (0.0–1.0). Failed runs remain at ``0.0``.", default=0.0, ) status: Optional[str] = Field( diff --git a/src/ContentProcessorWorkflow/src/steps/document_process/executor/document_process_executor.py b/src/ContentProcessorWorkflow/src/steps/document_process/executor/document_process_executor.py index f131c1a2..68a81b97 100644 --- a/src/ContentProcessorWorkflow/src/steps/document_process/executor/document_process_executor.py +++ b/src/ContentProcessorWorkflow/src/steps/document_process/executor/document_process_executor.py @@ -242,8 +242,12 @@ async def _on_poll(poll_data: dict) -> None: status_text = poll_result.get("status", "Failed") - schema_score_f = 0.0 - entity_score_f = 0.0 + # Failed / not-yet-scored documents default to ``0.0``; + # save_handler always emits numeric scores for Completed + # runs (probabilistic if available, otherwise structural + # completeness fallback). + schema_score_f: float = 0.0 + entity_score_f: float = 0.0 processed_time = "" result_payload = None @@ -253,18 +257,22 @@ async def _on_poll(poll_data: dict) -> None: ) if isinstance(final_payload, dict): status_text = final_payload.get("status") or status_text - try: - schema_score_f = float( - final_payload.get("schema_score") or 0.0 - ) - except Exception: - schema_score_f = 0.0 - try: - entity_score_f = float( - final_payload.get("entity_score") or 0.0 - ) - except Exception: - entity_score_f = 0.0 + + def _coerce_score(value: object) -> float: + """Coerce a raw score payload to ``float`` (default ``0.0``).""" + if value is None: + return 0.0 + try: + return float(value) + except (TypeError, ValueError): + return 0.0 + + schema_score_f = _coerce_score( + final_payload.get("schema_score") + ) + entity_score_f = _coerce_score( + final_payload.get("entity_score") + ) try: processed_time = ( final_payload.get("processed_time") or "" diff --git a/src/ContentProcessorWorkflow/src/utils/credential_util.py b/src/ContentProcessorWorkflow/src/utils/credential_util.py index b37de6d9..306fd180 100644 --- a/src/ContentProcessorWorkflow/src/utils/credential_util.py +++ b/src/ContentProcessorWorkflow/src/utils/credential_util.py @@ -19,7 +19,6 @@ from azure.identity import ( AzureCliCredential, AzureDeveloperCliCredential, - DefaultAzureCredential, ManagedIdentityCredential, ) from azure.identity import ( @@ -126,7 +125,12 @@ def get_azure_credential(): logging.info( "[AUTH] All CLI credentials failed - falling back to DefaultAzureCredential" ) - return DefaultAzureCredential() + + raise RuntimeError( + "No Azure authentication available. " + "Use Managed Identity in Azure or run " + "'az login' / 'azd auth login' locally." + ) def get_async_azure_credential(): diff --git a/src/ContentProcessorWorkflow/tests/unit/repositories/test_claim_process_model.py b/src/ContentProcessorWorkflow/tests/unit/repositories/test_claim_process_model.py index 195b9b36..a970555a 100644 --- a/src/ContentProcessorWorkflow/tests/unit/repositories/test_claim_process_model.py +++ b/src/ContentProcessorWorkflow/tests/unit/repositories/test_claim_process_model.py @@ -42,6 +42,7 @@ def test_defaults(self): assert cp.process_id == "p1" assert cp.file_name == "doc.pdf" assert cp.mime_type is None + # Defaults stay at ``0.0`` so failed/pre-save records render as 0%. assert cp.entity_score == 0.0 assert cp.schema_score == 0.0 assert cp.status is None @@ -57,6 +58,28 @@ def test_explicit_scores(self): assert cp.entity_score == 0.95 assert cp.schema_score == 0.87 + def test_explicit_zero_score_preserved(self): + """A literal ``0`` is a real score and must survive round-trip.""" + cp = Content_Process( + process_id="p1", + file_name="doc.pdf", + entity_score=0.0, + schema_score=0.0, + ) + assert cp.entity_score == 0.0 + assert cp.schema_score == 0.0 + + def test_failed_processing_keeps_default_zero(self): + """A failed file uses the ``0.0`` default so the UI renders ``0%``.""" + cp = Content_Process( + process_id="p1", + file_name="doc.pdf", + status="Failed", + ) + assert cp.status == "Failed" + assert cp.entity_score == 0.0 + assert cp.schema_score == 0.0 + # ── Claim_Process ──────────────────────────────────────────────────────────── diff --git a/src/ContentProcessorWorkflow/tests/unit/services/test_content_process_models.py b/src/ContentProcessorWorkflow/tests/unit/services/test_content_process_models.py index 19765025..c853c2d4 100644 --- a/src/ContentProcessorWorkflow/tests/unit/services/test_content_process_models.py +++ b/src/ContentProcessorWorkflow/tests/unit/services/test_content_process_models.py @@ -152,6 +152,17 @@ def test_construction_with_defaults(self): assert rec.id == "r1" assert rec.process_id == "" assert rec.status is None + # Defaults stay at ``0.0`` so failed/pre-save records render as 0% + # in the UI; save_handler overwrites with a real numeric score for + # Completed runs. + assert rec.entity_score == 0.0 + assert rec.schema_score == 0.0 + + def test_explicit_zero_score_preserved(self): + """A literal ``0.0`` must survive round-trip.""" + rec = ContentProcessRecord( + id="r1", process_id="r1", entity_score=0.0, schema_score=0.0 + ) assert rec.entity_score == 0.0 assert rec.schema_score == 0.0 diff --git a/src/ContentProcessorWorkflow/uv.lock b/src/ContentProcessorWorkflow/uv.lock index b7267abd..8b31bb03 100644 --- a/src/ContentProcessorWorkflow/uv.lock +++ b/src/ContentProcessorWorkflow/uv.lock @@ -712,14 +712,14 @@ wheels = [ [[package]] name = "authlib" -version = "1.6.11" +version = "1.6.12" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/28/10/b325d58ffe86815b399334a101e63bc6fa4e1953921cb23703b48a0a0220/authlib-1.6.11.tar.gz", hash = "sha256:64db35b9b01aeccb4715a6c9a6613a06f2bd7be2ab9d2eb89edd1dfc7580a38f", size = 165359, upload-time = "2026-04-16T07:22:50.279Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/30/6691fdc63b35f54a5a65e04fa1e59d827f4d4e8f4a39678ba7d3088ce0c8/authlib-1.6.12.tar.gz", hash = "sha256:0656d8482f28fc8221929d5f35b2bde5d13e10555ebc06b4561b0d622e83b1bd", size = 165368, upload-time = "2026-05-04T08:11:31.826Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/57/2f/55fca558f925a51db046e5b929deb317ddb05afed74b22d89f4eca578980/authlib-1.6.11-py2.py3-none-any.whl", hash = "sha256:c8687a9a26451c51a34a06fa17bb97cb15bba46a6a626755e2d7f50da8bff3e3", size = 244469, upload-time = "2026-04-16T07:22:48.413Z" }, + { url = "https://files.pythonhosted.org/packages/cd/51/9b0b5cd4cf683a02db937a6f9bbebcdc9c56558a7bb3763ce7d3512103c3/authlib-1.6.12-py2.py3-none-any.whl", hash = "sha256:e9229ad7fde610b139dd12f5edbe97eab9ee78bfb85691247e767727850b99ab", size = 244473, upload-time = "2026-05-04T08:11:30.354Z" }, ] [[package]] @@ -3228,7 +3228,7 @@ requires-dist = [ { name = "agent-framework", specifier = "==1.3.0" }, { name = "aiohttp", specifier = "==3.13.5" }, { name = "art", specifier = "==6.5" }, - { name = "authlib", specifier = "==1.6.11" }, + { name = "authlib", specifier = "==1.6.12" }, { name = "azure-ai-agents", specifier = "==1.2.0b5" }, { name = "azure-ai-inference", specifier = "==1.0.0b9" }, { name = "azure-ai-projects", specifier = "==2.1.0" }, diff --git a/src/tests/ContentProcessor/utils/test_azure_credential_utils.py b/src/tests/ContentProcessor/utils/test_azure_credential_utils.py index 216b302e..7f0f88a0 100644 --- a/src/tests/ContentProcessor/utils/test_azure_credential_utils.py +++ b/src/tests/ContentProcessor/utils/test_azure_credential_utils.py @@ -7,6 +7,8 @@ from unittest.mock import MagicMock, patch +import pytest + import libs.utils.azure_credential_utils as azure_credential_utils MODULE = "libs.utils.azure_credential_utils" @@ -45,16 +47,16 @@ def test_returns_user_assigned_with_client_id(self, mock_managed): mock_managed.assert_called_once_with(client_id="test-client-id") assert credential == mock_instance - @patch(f"{MODULE}.DefaultAzureCredential") @patch(f"{MODULE}.AzureDeveloperCliCredential", side_effect=Exception("no azd")) @patch(f"{MODULE}.AzureCliCredential", side_effect=Exception("no az")) @patch.dict("os.environ", {}, clear=True) - def test_falls_back_to_default(self, mock_cli, mock_dev_cli, mock_default): - mock_instance = MagicMock() - mock_default.return_value = mock_instance - credential = azure_credential_utils.get_azure_credential() - mock_default.assert_called_once() - assert credential == mock_instance + def test_raises_when_no_credentials_available( + self, mock_cli, mock_dev_cli + ): + with pytest.raises(RuntimeError) as exc: + azure_credential_utils.get_azure_credential() + + assert "No Azure authentication available" in str(exc.value) # ── TestGetAsyncAzureCredential ───────────────────────────────────────── diff --git a/src/tests/ContentProcessor/utils/test_azure_credential_utils_extended.py b/src/tests/ContentProcessor/utils/test_azure_credential_utils_extended.py index 11858fdc..edd735d8 100644 --- a/src/tests/ContentProcessor/utils/test_azure_credential_utils_extended.py +++ b/src/tests/ContentProcessor/utils/test_azure_credential_utils_extended.py @@ -42,26 +42,22 @@ def test_get_azure_credential_with_website_site_name(self, monkeypatch): assert credential == mock_instance def test_get_azure_credential_cli_failure_fallback(self, monkeypatch): - """Test fallback to DefaultAzureCredential when CLI credentials fail""" + """Test RuntimeError when all credential options fail""" # Clear all Azure environment indicators for key in ["WEBSITE_SITE_NAME", "AZURE_CLIENT_ID", "MSI_ENDPOINT", "IDENTITY_ENDPOINT", "KUBERNETES_SERVICE_HOST", "CONTAINER_REGISTRY_LOGIN"]: monkeypatch.delenv(key, raising=False) with patch('libs.utils.azure_credential_utils.AzureCliCredential') as mock_cli_cred, \ - patch('libs.utils.azure_credential_utils.AzureDeveloperCliCredential') as mock_azd_cred, \ - patch('libs.utils.azure_credential_utils.DefaultAzureCredential') as mock_default: + patch('libs.utils.azure_credential_utils.AzureDeveloperCliCredential') as mock_azd_cred: - # Make both CLI credentials raise exceptions mock_cli_cred.side_effect = Exception("CLI credential failed") mock_azd_cred.side_effect = Exception("AZD credential failed") - mock_default_instance = Mock() - mock_default.return_value = mock_default_instance - credential = get_azure_credential() + with pytest.raises(RuntimeError) as exc: + get_azure_credential() - assert credential == mock_default_instance - mock_default.assert_called_once() + assert "No Azure authentication available" in str(exc.value) def test_get_azure_credential_azd_success(self, monkeypatch): """Test successful Azure Developer CLI credential""" diff --git a/src/tests/ContentProcessorWorkflow/repositories/test_claim_process_model.py b/src/tests/ContentProcessorWorkflow/repositories/test_claim_process_model.py index 36de49c0..fee05fb1 100644 --- a/src/tests/ContentProcessorWorkflow/repositories/test_claim_process_model.py +++ b/src/tests/ContentProcessorWorkflow/repositories/test_claim_process_model.py @@ -42,6 +42,7 @@ def test_defaults(self): assert cp.process_id == "p1" assert cp.file_name == "doc.pdf" assert cp.mime_type is None + # Defaults stay at ``0.0`` so failed/pre-save records render as 0%. assert cp.entity_score == 0.0 assert cp.schema_score == 0.0 assert cp.status is None diff --git a/src/tests/ContentProcessorWorkflow/services/test_content_process_models.py b/src/tests/ContentProcessorWorkflow/services/test_content_process_models.py index 059b2938..5133852a 100644 --- a/src/tests/ContentProcessorWorkflow/services/test_content_process_models.py +++ b/src/tests/ContentProcessorWorkflow/services/test_content_process_models.py @@ -218,6 +218,8 @@ def test_content_process_record_defaults(self): assert record.process_id == "" assert record.processed_file_name is None assert record.processed_file_mime_type is None + # Defaults stay at ``0.0`` so failed/pre-save records render as 0% + # in the UI. assert record.entity_score == 0.0 assert record.schema_score == 0.0 diff --git a/src/tests/ContentProcessorWorkflow/utils/test_credential_util_extended.py b/src/tests/ContentProcessorWorkflow/utils/test_credential_util_extended.py index d4fda81d..40cfaf68 100644 --- a/src/tests/ContentProcessorWorkflow/utils/test_credential_util_extended.py +++ b/src/tests/ContentProcessorWorkflow/utils/test_credential_util_extended.py @@ -1,5 +1,6 @@ """Extended tests for credential_util.py to improve coverage""" from unittest.mock import Mock, patch +import pytest from utils.credential_util import ( get_azure_credential, get_async_azure_credential, @@ -40,24 +41,27 @@ def test_get_azure_credential_app_service_environment(self, monkeypatch): assert credential == mock_instance def test_get_azure_credential_all_cli_fail(self, monkeypatch): - """Test fallback when all CLI credentials fail""" - for key in ["WEBSITE_SITE_NAME", "AZURE_CLIENT_ID", "MSI_ENDPOINT", - "IDENTITY_ENDPOINT", "KUBERNETES_SERVICE_HOST", "CONTAINER_REGISTRY_LOGIN"]: + """Test RuntimeError when all credential options fail""" + for key in [ + "WEBSITE_SITE_NAME", + "AZURE_CLIENT_ID", + "MSI_ENDPOINT", + "IDENTITY_ENDPOINT", + "KUBERNETES_SERVICE_HOST", + "CONTAINER_REGISTRY_LOGIN", + ]: monkeypatch.delenv(key, raising=False) with patch('utils.credential_util.AzureCliCredential') as mock_cli, \ - patch('utils.credential_util.AzureDeveloperCliCredential') as mock_azd, \ - patch('utils.credential_util.DefaultAzureCredential') as mock_default: + patch('utils.credential_util.AzureDeveloperCliCredential') as mock_azd: mock_cli.side_effect = Exception("AzureCLI not available") mock_azd.side_effect = Exception("AzureDeveloperCLI not available") - mock_default_instance = Mock() - mock_default.return_value = mock_default_instance - credential = get_azure_credential() + with pytest.raises(RuntimeError) as exc: + get_azure_credential() - assert credential == mock_default_instance - mock_default.assert_called_once() + assert "No Azure authentication available" in str(exc.value) def test_get_azure_credential_cli_success(self, monkeypatch): """Test successful Azure CLI credential"""