LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3430 [FIX] Update modified_at field correctly for models (#1928)
Chandrasekharan Mfd5929083e8b
diff --git a/backend/adapter_processor_v2/models.py b/backend/adapter_processor_v2/models.pyindex 13be07cdc..a6fab0c1f 100644--- a/backend/adapter_processor_v2/models.py+++ b/backend/adapter_processor_v2/models.py@@ -10,7 +10,7 @@ from django.db import modelsfrom django.db.models import QuerySetfrom tenant_account_v2.models import OrganizationMemberfrom utils.exceptions import InvalidEncryptionKey-from utils.models.base_model import BaseModel+from utils.models.base_model import BaseModel, BaseModelManagerfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,DefaultOrganizationMixin,@@ -29,7 +29,7 @@ ADAPTER_ID_LENGTH = 128logger = logging.getLogger(__name__)-class AdapterInstanceModelManager(DefaultOrganizationManagerMixin, models.Manager):+class AdapterInstanceModelManager(DefaultOrganizationManagerMixin, BaseModelManager):def get_queryset(self) -> QuerySet[Any]:return super().get_queryset()diff --git a/backend/api_v2/models.py b/backend/api_v2/models.pyindex 532b67d2d..cc19902bd 100644--- a/backend/api_v2/models.py+++ b/backend/api_v2/models.py@@ -7,7 +7,7 @@ from django.db import modelsfrom django.db.models.signals import post_deletefrom django.dispatch import receiverfrom pipeline_v2.models import Pipeline-from utils.models.base_model import BaseModel+from utils.models.base_model import BaseModel, BaseModelManagerfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,DefaultOrganizationMixin,@@ -24,7 +24,7 @@ DESCRIPTION_MAX_LENGTH = 255API_ENDPOINT_MAX_LENGTH = 255-class APIDeploymentModelManager(DefaultOrganizationManagerMixin, models.Manager):+class APIDeploymentModelManager(DefaultOrganizationManagerMixin, BaseModelManager):def for_user(self, user):"""Filter API deployments that the user can access:- API deployments created by the userdiff --git a/backend/connector_v2/models.py b/backend/connector_v2/models.pyindex c3d0e6108..73ea38b57 100644--- a/backend/connector_v2/models.py+++ b/backend/connector_v2/models.py@@ -8,7 +8,7 @@ from connector_processor.connector_processor import ConnectorProcessorfrom connector_processor.constants import ConnectorKeysfrom django.db import modelsfrom utils.fields import EncryptedBinaryField-from utils.models.base_model import BaseModel+from utils.models.base_model import BaseModel, BaseModelManagerfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,DefaultOrganizationMixin,@@ -22,7 +22,7 @@ VERSION_NAME_SIZE = 64logger = logging.getLogger(__name__)-class ConnectorInstanceModelManager(DefaultOrganizationManagerMixin, models.Manager):+class ConnectorInstanceModelManager(DefaultOrganizationManagerMixin, BaseModelManager):def get_queryset(self) -> models.QuerySet:return super().get_queryset()diff --git a/backend/dashboard_metrics/management/commands/backfill_metrics.py b/backend/dashboard_metrics/management/commands/backfill_metrics.pyindex daf0b3130..9c4d82bac 100644--- a/backend/dashboard_metrics/management/commands/backfill_metrics.py+++ b/backend/dashboard_metrics/management/commands/backfill_metrics.py@@ -424,7 +424,7 @@ class Command(BaseCommand):"project","tag",],- update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],+ update_fields=["metric_type", "metric_value", "metric_count"],)return len(objects)@@ -457,7 +457,7 @@ class Command(BaseCommand):"project","tag",],- update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],+ update_fields=["metric_type", "metric_value", "metric_count"],)return len(objects)@@ -490,6 +490,6 @@ class Command(BaseCommand):"project","tag",],- update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],+ update_fields=["metric_type", "metric_value", "metric_count"],)return len(objects)diff --git a/backend/dashboard_metrics/models.py b/backend/dashboard_metrics/models.pyindex 44fec5439..2bc4baf6a 100644--- a/backend/dashboard_metrics/models.py+++ b/backend/dashboard_metrics/models.py@@ -3,7 +3,7 @@import uuidfrom django.db import models-from utils.models.base_model import BaseModel+from utils.models.base_model import BaseModel, BaseModelManagerfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,DefaultOrganizationMixin,@@ -25,19 +25,19 @@ class MetricType(models.TextChoices):HISTOGRAM = "histogram", "Histogram"-class EventMetricsHourlyManager(DefaultOrganizationManagerMixin):+class EventMetricsHourlyManager(DefaultOrganizationManagerMixin, BaseModelManager):"""Manager for EventMetricsHourly with organization filtering."""pass-class EventMetricsDailyManager(DefaultOrganizationManagerMixin):+class EventMetricsDailyManager(DefaultOrganizationManagerMixin, BaseModelManager):"""Manager for EventMetricsDaily with organization filtering."""pass-class EventMetricsMonthlyManager(DefaultOrganizationManagerMixin):+class EventMetricsMonthlyManager(DefaultOrganizationManagerMixin, BaseModelManager):"""Manager for EventMetricsMonthly with organization filtering."""passdiff --git a/backend/dashboard_metrics/tasks.py b/backend/dashboard_metrics/tasks.pyindex 3246e7e46..181c98513 100644--- a/backend/dashboard_metrics/tasks.py+++ b/backend/dashboard_metrics/tasks.py@@ -121,7 +121,7 @@ def _bulk_upsert_hourly(aggregations: dict) -> int:objects,update_conflicts=True,unique_fields=["organization", "timestamp", "metric_name", "project", "tag"],- update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],+ update_fields=["metric_type", "metric_value", "metric_count"],)return len(objects)@@ -160,7 +160,7 @@ def _bulk_upsert_daily(aggregations: dict) -> int:objects,update_conflicts=True,unique_fields=["organization", "date", "metric_name", "project", "tag"],- update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],+ update_fields=["metric_type", "metric_value", "metric_count"],)return len(objects)@@ -199,7 +199,7 @@ def _bulk_upsert_monthly(aggregations: dict) -> int:objects,update_conflicts=True,unique_fields=["organization", "month", "metric_name", "project", "tag"],- update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],+ update_fields=["metric_type", "metric_value", "metric_count"],)return len(objects)diff --git a/backend/pipeline_v2/models.py b/backend/pipeline_v2/models.pyindex 00b12c848..65fb5257a 100644--- a/backend/pipeline_v2/models.py+++ b/backend/pipeline_v2/models.py@@ -4,7 +4,7 @@ from account_v2.models import Userfrom django.conf import settingsfrom django.db import modelsfrom django.db.models import Q-from utils.models.base_model import BaseModel+from utils.models.base_model import BaseModel, BaseModelManagerfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,DefaultOrganizationMixin,@@ -18,7 +18,7 @@ APP_ID_LENGTH = 32PIPELINE_NAME_LENGTH = 32-class PipelineModelManager(DefaultOrganizationManagerMixin, models.Manager):+class PipelineModelManager(DefaultOrganizationManagerMixin, BaseModelManager):def for_user(self, user):"""Filter pipelines that the user can access:- Pipelines created by the userdiff --git a/backend/prompt_studio/prompt_studio_core_v2/models.py b/backend/prompt_studio/prompt_studio_core_v2/models.pyindex 406c157ef..1e1802b77 100644--- a/backend/prompt_studio/prompt_studio_core_v2/models.py+++ b/backend/prompt_studio/prompt_studio_core_v2/models.py@@ -8,7 +8,7 @@ from django.db import modelsfrom django.db.models import QuerySetfrom utils.file_storage.constants import FileStorageKeysfrom utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper-from utils.models.base_model import BaseModel+from utils.models.base_model import BaseModel, BaseModelManagerfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,DefaultOrganizationMixin,@@ -21,7 +21,7 @@ from unstract.sdk1.file_storage.env_helper import EnvHelperlogger = logging.getLogger(__name__)-class CustomToolModelManager(DefaultOrganizationManagerMixin, models.Manager):+class CustomToolModelManager(DefaultOrganizationManagerMixin, BaseModelManager):def for_user(self, user: User) -> QuerySet[Any]:if getattr(user, "is_service_account", False):return self.all()diff --git a/backend/prompt_studio/prompt_studio_registry_v2/models.py b/backend/prompt_studio/prompt_studio_registry_v2/models.pyindex c4cd54cc0..34478c8e7 100644--- a/backend/prompt_studio/prompt_studio_registry_v2/models.py+++ b/backend/prompt_studio/prompt_studio_registry_v2/models.py@@ -5,7 +5,7 @@ from typing import Anyfrom account_v2.models import Userfrom django.db import modelsfrom django.db.models import QuerySet-from utils.models.base_model import BaseModel+from utils.models.base_model import BaseModel, BaseModelManagerfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,DefaultOrganizationMixin,@@ -21,7 +21,7 @@ from prompt_studio.prompt_studio_v2.models import CustomToollogger = logging.getLogger(__name__)-class PromptStudioRegistryModelManager(DefaultOrganizationManagerMixin, models.Manager):+class PromptStudioRegistryModelManager(DefaultOrganizationManagerMixin, BaseModelManager):def get_queryset(self) -> QuerySet[Any]:return super().get_queryset()diff --git a/backend/tags/models.py b/backend/tags/models.pyindex 94c5c8cf5..7f366152b 100644--- a/backend/tags/models.py+++ b/backend/tags/models.py@@ -1,7 +1,7 @@import uuidfrom django.db import models-from utils.models.base_model import BaseModel+from utils.models.base_model import BaseModel, BaseModelManagerfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,DefaultOrganizationMixin,@@ -9,7 +9,7 @@ from utils.models.organization_mixin import (from utils.user_context import UserContext-class TagModelManager(DefaultOrganizationManagerMixin, models.Manager):+class TagModelManager(DefaultOrganizationManagerMixin, BaseModelManager):def get_or_create_tags(self, tag_names: list[str]) -> list["Tag"]:"""Retrieves or creates tags based on a list of tag names.diff --git a/backend/tool_instance_v2/models.py b/backend/tool_instance_v2/models.pyindex 971f1249b..d858da232 100644--- a/backend/tool_instance_v2/models.py+++ b/backend/tool_instance_v2/models.py@@ -3,7 +3,7 @@ import uuidfrom account_v2.models import Userfrom django.db import modelsfrom django.db.models import QuerySet-from utils.models.base_model import BaseModel+from utils.models.base_model import BaseModel, BaseModelManagerfrom workflow_manager.workflow_v2.models.workflow import WorkflowTOOL_ID_LENGTH = 64@@ -11,7 +11,7 @@ TOOL_VERSION_LENGTH = 16TOOL_STATUS_LENGTH = 32-class ToolInstanceManager(models.Manager):+class ToolInstanceManager(BaseModelManager):def get_instances_for_workflow(self, workflow: uuid.UUID) -> QuerySet["ToolInstance"]:return self.filter(workflow=workflow)diff --git a/backend/usage_v2/models.py b/backend/usage_v2/models.pyindex 8da3d751b..57ae3d143 100644--- a/backend/usage_v2/models.py+++ b/backend/usage_v2/models.py@@ -1,7 +1,7 @@import uuidfrom django.db import models-from utils.models.base_model import BaseModel+from utils.models.base_model import BaseModel, BaseModelManagerfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,DefaultOrganizationMixin,@@ -19,7 +19,7 @@ class LLMUsageReason(models.TextChoices):SUMMARIZE = "summarize", "Summarize"-class UsageModelManager(DefaultOrganizationManagerMixin, models.Manager):+class UsageModelManager(DefaultOrganizationManagerMixin, BaseModelManager):passdiff --git a/backend/utils/models/base_model.py b/backend/utils/models/base_model.pyindex b26f0f67d..c2b288b9e 100644--- a/backend/utils/models/base_model.py+++ b/backend/utils/models/base_model.py@@ -1,9 +1,122 @@from django.db import models+from django.utils import timezone++_AUTO_NOW_FIELD = "modified_at"+++def _with_modified_at(fields):+ """Return a new list containing ``fields`` plus ``modified_at`` if absent.++ Centralises the "inject modified_at into a partial field list" rule so+ ``bulk_update``, ``bulk_create`` and ``BaseModel.save`` apply it the same+ way.+ """+ fields = list(fields)+ if _AUTO_NOW_FIELD not in fields:+ fields.append(_AUTO_NOW_FIELD)+ return fields+++class BaseModelQuerySet(models.QuerySet):+ """QuerySet that mirrors ``auto_now`` semantics for bulk update paths.++ ``modified_at = models.DateTimeField(auto_now=True)`` only fires on+ ``Model.save()``. ``QuerySet.update()`` and ``QuerySet.bulk_update()``+ issue raw SQL and bypass ``save()``, leaving ``modified_at`` at whatever+ value it had before the bulk path ran (creation time for never-saved+ rows, the previous save() timestamp for others) — silently drifting the+ audit trail. This QuerySet patches both paths so callers don't have to+ remember.++ Callers can still override by passing ``modified_at`` explicitly (or by+ including ``modified_at`` in the ``fields`` list for ``bulk_update``).++ Note: this is a manager-level convention, not a model-level guarantee.+ Subclasses that reassign ``objects`` to a plain ``models.Manager``, raw+ SQL, and migration-time models returned by ``apps.get_model()`` all+ bypass these overrides.+ """++ def update(self, **kwargs):+ kwargs.setdefault(_AUTO_NOW_FIELD, timezone.now())+ return super().update(**kwargs)++ def bulk_update(self, objs, fields, *args, **kwargs):+ # Stamp modified_at on each obj only when the caller didn't list it;+ # materialize objs first because we iterate the sequence twice (once+ # to stamp, once via super()) and a generator would be exhausted.+ if _AUTO_NOW_FIELD not in fields:+ objs = list(objs)+ now = timezone.now()+ for obj in objs:+ obj.modified_at = now+ fields = _with_modified_at(fields)+ return super().bulk_update(objs, fields, *args, **kwargs)++ def bulk_create(+ self, objs, *args, update_conflicts=False, update_fields=None, **kwargs+ ):+ # On upsert-on-conflict Django runs an UPDATE with only the listed+ # fields, which skips auto_now the same way save(update_fields=...)+ # does. Insert-only bulk_create already handles auto_now itself.+ if update_conflicts and update_fields:+ update_fields = _with_modified_at(update_fields)+ return super().bulk_create(+ objs,+ *args,+ update_conflicts=update_conflicts,+ update_fields=update_fields,+ **kwargs,+ )+++BaseModelManager = models.Manager.from_queryset(BaseModelQuerySet)class BaseModel(models.Model):+ """Abstract base with managed ``created_at`` / ``modified_at`` timestamps.++ Subclasses inherit ``BaseModelManager`` as the default manager, which+ auto-bumps ``modified_at`` on ``QuerySet.update()``, ``bulk_update()``+ and upsert-mode ``bulk_create()``. The ``save()`` override below does+ the same for partial ``save(update_fields=[...])`` calls.++ Subclasses that need a custom manager should compose ``BaseModelManager``+ (e.g. ``class FooManager(MyMixin, BaseModelManager)``) — otherwise the+ auto-bump on bulk paths is silently lost.+ """+created_at = models.DateTimeField(auto_now_add=True)modified_at = models.DateTimeField(auto_now=True)+ objects = BaseModelManager()+class Meta:abstract = True++ def save(+ self,+ force_insert=False,+ force_update=False,+ using=None,+ update_fields=None,+ **kwargs,+ ):+ # Django's save(update_fields=...) only writes the listed columns.+ # auto_now still updates modified_at on the in-memory instance, but+ # the new value is never persisted unless modified_at is in+ # update_fields. Auto-include it so partial saves don't silently drop+ # the bump. Preserve Django's documented no-op semantics for+ # update_fields=[] (signals-only save, no column writes).+ #+ # Signature mirrors Django's positional order so callers passing+ # force_insert/force_update positionally still hit this override.+ if update_fields:+ update_fields = _with_modified_at(update_fields)+ return super().save(+ force_insert=force_insert,+ force_update=force_update,+ using=using,+ update_fields=update_fields,+ **kwargs,+ )diff --git a/backend/utils/models/org_aware_manager.py b/backend/utils/models/org_aware_manager.pyindex 3ec6e70bd..ab40303db 100644--- a/backend/utils/models/org_aware_manager.py+++ b/backend/utils/models/org_aware_manager.py@@ -3,15 +3,15 @@import loggingfrom django.core.exceptions import ImproperlyConfigured-from django.db import modelsfrom django.db.utils import OperationalError, ProgrammingError+from utils.models.base_model import BaseModelManagerfrom utils.models.org_path_discovery import get_org_pathfrom utils.user_context import UserContextlogger = logging.getLogger(__name__)-class OrgAwareManager(models.Manager):+class OrgAwareManager(BaseModelManager):"""Manager that auto-discovers FK path to Organization and appliesorg filtering to all queries in request context.diff --git a/backend/workflow_manager/endpoint_v2/models.py b/backend/workflow_manager/endpoint_v2/models.pyindex e07737ba7..44742dfc3 100644--- a/backend/workflow_manager/endpoint_v2/models.py+++ b/backend/workflow_manager/endpoint_v2/models.py@@ -2,12 +2,12 @@ import uuidfrom connector_v2.models import ConnectorInstancefrom django.db import models-from utils.models.base_model import BaseModel+from utils.models.base_model import BaseModel, BaseModelManagerfrom utils.user_context import UserContextfrom workflow_manager.workflow_v2.models.workflow import Workflow-class WorkflowEndpointModelManager(models.Manager):+class WorkflowEndpointModelManager(BaseModelManager):def get_queryset(self):# Validating organizationorganization = UserContext.get_organization()diff --git a/backend/workflow_manager/file_execution/models.py b/backend/workflow_manager/file_execution/models.pyindex d5106855e..105b3875a 100644--- a/backend/workflow_manager/file_execution/models.py+++ b/backend/workflow_manager/file_execution/models.py@@ -4,7 +4,7 @@ from typing import Anyfrom django.db import modelsfrom utils.common_utils import CommonUtils-from utils.models.base_model import BaseModel+from utils.models.base_model import BaseModel, BaseModelManagerfrom workflow_manager.endpoint_v2.dto import FileHashfrom workflow_manager.workflow_v2.enums import ExecutionStatus@@ -15,7 +15,7 @@ HASH_LENGTH = 64MIME_TYPE_LENGTH = 128-class WorkflowFileExecutionManager(models.Manager):+class WorkflowFileExecutionManager(BaseModelManager):def get_or_create_file_execution(self,workflow_execution: Any,diff --git a/backend/workflow_manager/internal_views.py b/backend/workflow_manager/internal_views.pyindex d11ae5617..c822e5e7b 100644--- a/backend/workflow_manager/internal_views.py+++ b/backend/workflow_manager/internal_views.py@@ -7,7 +7,6 @@ import uuidfrom django.db import transactionfrom django.shortcuts import get_object_or_404-from django.utils import timezonefrom rest_framework import status, viewsetsfrom rest_framework.decorators import actionfrom rest_framework.response import Response@@ -2009,7 +2008,6 @@ class BatchStatusUpdateAPIView(APIView):if update.get("execution_time") is not None:execution.execution_time = update["execution_time"]- execution.modified_at = timezone.now()execution.save()successful_updates.append(diff --git a/backend/workflow_manager/workflow_v2/file_history_helper.py b/backend/workflow_manager/workflow_v2/file_history_helper.pyindex 12b0a9c2c..687f88dfe 100644--- a/backend/workflow_manager/workflow_v2/file_history_helper.py+++ b/backend/workflow_manager/workflow_v2/file_history_helper.py@@ -288,7 +288,6 @@ class FileHistoryHelper:result=str(result),metadata=FileHistoryHelper._safe_str(metadata),error=FileHistoryHelper._safe_str(error),- modified_at=timezone.now(),)# Refresh from DB to get updated valuesfile_history.refresh_from_db()diff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.pyindex 93623118d..45886bd64 100644--- a/backend/workflow_manager/workflow_v2/models/execution.py+++ b/backend/workflow_manager/workflow_v2/models/execution.py@@ -12,7 +12,7 @@ from usage_v2.constants import UsageKeysfrom usage_v2.helper import UsageHelperfrom usage_v2.models import Usagefrom utils.common_utils import CommonUtils-from utils.models.base_model import BaseModel+from utils.models.base_model import BaseModel, BaseModelManagerfrom workflow_manager.execution.dto import ExecutionCachefrom workflow_manager.execution.execution_cache_utils import ExecutionCacheUtils@@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)EXECUTION_ERROR_LENGTH = 256-class WorkflowExecutionManager(models.Manager):+class WorkflowExecutionManager(BaseModelManager):"""Custom manager for WorkflowExecution model to handle user-specific filtering."""def for_user(self, user) -> QuerySet:diff --git a/backend/workflow_manager/workflow_v2/models/workflow.py b/backend/workflow_manager/workflow_v2/models/workflow.pyindex dd945fc7f..0029f9599 100644--- a/backend/workflow_manager/workflow_v2/models/workflow.py+++ b/backend/workflow_manager/workflow_v2/models/workflow.py@@ -4,7 +4,7 @@ from account_v2.models import Userfrom django.conf import settingsfrom django.core.validators import MinValueValidatorfrom django.db import models-from utils.models.base_model import BaseModel+from utils.models.base_model import BaseModel, BaseModelManagerfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,DefaultOrganizationMixin,@@ -15,7 +15,7 @@ DESCRIPTION_FIELD_LENGTH = 490WORKFLOW_NAME_SIZE = 128-class WorkflowModelManager(DefaultOrganizationManagerMixin, models.Manager):+class WorkflowModelManager(DefaultOrganizationManagerMixin, BaseModelManager):def for_user(self, user):"""Filter workflows that the user can access:- Workflows created by the userdiff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.pyindex dc428d948..629b52bb2 100644--- a/backend/workflow_manager/workflow_v2/views.py+++ b/backend/workflow_manager/workflow_v2/views.py@@ -6,7 +6,6 @@ from django.conf import settingsfrom django.db import transactionfrom django.db.models.query import QuerySetfrom django.shortcuts import get_object_or_404-from django.utils import timezonefrom django.views.decorators.csrf import csrf_exemptfrom permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrgfrom pipeline_v2.models import Pipeline@@ -528,7 +527,6 @@ class WorkflowExecutionInternalViewSet(viewsets.ReadOnlyModelViewSet):if validated_data.get("execution_time") is not None:execution.execution_time = validated_data["execution_time"]- execution.modified_at = timezone.now()execution.save()logger.info(