Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-3430 [FIX] Update modified_at field correctly for models (#1928)

Chandrasekharan Mfd5929083e8b
diff --git a/backend/adapter_processor_v2/models.py b/backend/adapter_processor_v2/models.py
index 13be07cdc..a6fab0c1f 100644
--- a/backend/adapter_processor_v2/models.py
+++ b/backend/adapter_processor_v2/models.py
@@ -10,7 +10,7 @@ from django.db import models
from django.db.models import QuerySet
from tenant_account_v2.models import OrganizationMember
from utils.exceptions import InvalidEncryptionKey
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
from utils.models.organization_mixin import (
DefaultOrganizationManagerMixin,
DefaultOrganizationMixin,
@@ -29,7 +29,7 @@ ADAPTER_ID_LENGTH = 128
logger = logging.getLogger(__name__)
-class AdapterInstanceModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class AdapterInstanceModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
def get_queryset(self) -> QuerySet[Any]:
return super().get_queryset()
diff --git a/backend/api_v2/models.py b/backend/api_v2/models.py
index 532b67d2d..cc19902bd 100644
--- a/backend/api_v2/models.py
+++ b/backend/api_v2/models.py
@@ -7,7 +7,7 @@ from django.db import models
from django.db.models.signals import post_delete
from django.dispatch import receiver
from pipeline_v2.models import Pipeline
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
from utils.models.organization_mixin import (
DefaultOrganizationManagerMixin,
DefaultOrganizationMixin,
@@ -24,7 +24,7 @@ DESCRIPTION_MAX_LENGTH = 255
API_ENDPOINT_MAX_LENGTH = 255
-class APIDeploymentModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class APIDeploymentModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
def for_user(self, user):
"""Filter API deployments that the user can access:
- API deployments created by the user
diff --git a/backend/connector_v2/models.py b/backend/connector_v2/models.py
index c3d0e6108..73ea38b57 100644
--- a/backend/connector_v2/models.py
+++ b/backend/connector_v2/models.py
@@ -8,7 +8,7 @@ from connector_processor.connector_processor import ConnectorProcessor
from connector_processor.constants import ConnectorKeys
from django.db import models
from utils.fields import EncryptedBinaryField
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
from utils.models.organization_mixin import (
DefaultOrganizationManagerMixin,
DefaultOrganizationMixin,
@@ -22,7 +22,7 @@ VERSION_NAME_SIZE = 64
logger = logging.getLogger(__name__)
-class ConnectorInstanceModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class ConnectorInstanceModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
def get_queryset(self) -> models.QuerySet:
return super().get_queryset()
diff --git a/backend/dashboard_metrics/management/commands/backfill_metrics.py b/backend/dashboard_metrics/management/commands/backfill_metrics.py
index daf0b3130..9c4d82bac 100644
--- a/backend/dashboard_metrics/management/commands/backfill_metrics.py
+++ b/backend/dashboard_metrics/management/commands/backfill_metrics.py
@@ -424,7 +424,7 @@ class Command(BaseCommand):
"project",
"tag",
],
- update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],
+ update_fields=["metric_type", "metric_value", "metric_count"],
)
return len(objects)
@@ -457,7 +457,7 @@ class Command(BaseCommand):
"project",
"tag",
],
- update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],
+ update_fields=["metric_type", "metric_value", "metric_count"],
)
return len(objects)
@@ -490,6 +490,6 @@ class Command(BaseCommand):
"project",
"tag",
],
- update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],
+ update_fields=["metric_type", "metric_value", "metric_count"],
)
return len(objects)
diff --git a/backend/dashboard_metrics/models.py b/backend/dashboard_metrics/models.py
index 44fec5439..2bc4baf6a 100644
--- a/backend/dashboard_metrics/models.py
+++ b/backend/dashboard_metrics/models.py
@@ -3,7 +3,7 @@
import uuid
from django.db import models
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
from utils.models.organization_mixin import (
DefaultOrganizationManagerMixin,
DefaultOrganizationMixin,
@@ -25,19 +25,19 @@ class MetricType(models.TextChoices):
HISTOGRAM = "histogram", "Histogram"
-class EventMetricsHourlyManager(DefaultOrganizationManagerMixin):
+class EventMetricsHourlyManager(DefaultOrganizationManagerMixin, BaseModelManager):
"""Manager for EventMetricsHourly with organization filtering."""
pass
-class EventMetricsDailyManager(DefaultOrganizationManagerMixin):
+class EventMetricsDailyManager(DefaultOrganizationManagerMixin, BaseModelManager):
"""Manager for EventMetricsDaily with organization filtering."""
pass
-class EventMetricsMonthlyManager(DefaultOrganizationManagerMixin):
+class EventMetricsMonthlyManager(DefaultOrganizationManagerMixin, BaseModelManager):
"""Manager for EventMetricsMonthly with organization filtering."""
pass
diff --git a/backend/dashboard_metrics/tasks.py b/backend/dashboard_metrics/tasks.py
index 3246e7e46..181c98513 100644
--- a/backend/dashboard_metrics/tasks.py
+++ b/backend/dashboard_metrics/tasks.py
@@ -121,7 +121,7 @@ def _bulk_upsert_hourly(aggregations: dict) -> int:
objects,
update_conflicts=True,
unique_fields=["organization", "timestamp", "metric_name", "project", "tag"],
- update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],
+ update_fields=["metric_type", "metric_value", "metric_count"],
)
return len(objects)
@@ -160,7 +160,7 @@ def _bulk_upsert_daily(aggregations: dict) -> int:
objects,
update_conflicts=True,
unique_fields=["organization", "date", "metric_name", "project", "tag"],
- update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],
+ update_fields=["metric_type", "metric_value", "metric_count"],
)
return len(objects)
@@ -199,7 +199,7 @@ def _bulk_upsert_monthly(aggregations: dict) -> int:
objects,
update_conflicts=True,
unique_fields=["organization", "month", "metric_name", "project", "tag"],
- update_fields=["metric_type", "metric_value", "metric_count", "modified_at"],
+ update_fields=["metric_type", "metric_value", "metric_count"],
)
return len(objects)
diff --git a/backend/pipeline_v2/models.py b/backend/pipeline_v2/models.py
index 00b12c848..65fb5257a 100644
--- a/backend/pipeline_v2/models.py
+++ b/backend/pipeline_v2/models.py
@@ -4,7 +4,7 @@ from account_v2.models import User
from django.conf import settings
from django.db import models
from django.db.models import Q
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
from utils.models.organization_mixin import (
DefaultOrganizationManagerMixin,
DefaultOrganizationMixin,
@@ -18,7 +18,7 @@ APP_ID_LENGTH = 32
PIPELINE_NAME_LENGTH = 32
-class PipelineModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class PipelineModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
def for_user(self, user):
"""Filter pipelines that the user can access:
- Pipelines created by the user
diff --git a/backend/prompt_studio/prompt_studio_core_v2/models.py b/backend/prompt_studio/prompt_studio_core_v2/models.py
index 406c157ef..1e1802b77 100644
--- a/backend/prompt_studio/prompt_studio_core_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_core_v2/models.py
@@ -8,7 +8,7 @@ from django.db import models
from django.db.models import QuerySet
from utils.file_storage.constants import FileStorageKeys
from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
from utils.models.organization_mixin import (
DefaultOrganizationManagerMixin,
DefaultOrganizationMixin,
@@ -21,7 +21,7 @@ from unstract.sdk1.file_storage.env_helper import EnvHelper
logger = logging.getLogger(__name__)
-class CustomToolModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class CustomToolModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
def for_user(self, user: User) -> QuerySet[Any]:
if getattr(user, "is_service_account", False):
return self.all()
diff --git a/backend/prompt_studio/prompt_studio_registry_v2/models.py b/backend/prompt_studio/prompt_studio_registry_v2/models.py
index c4cd54cc0..34478c8e7 100644
--- a/backend/prompt_studio/prompt_studio_registry_v2/models.py
+++ b/backend/prompt_studio/prompt_studio_registry_v2/models.py
@@ -5,7 +5,7 @@ from typing import Any
from account_v2.models import User
from django.db import models
from django.db.models import QuerySet
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
from utils.models.organization_mixin import (
DefaultOrganizationManagerMixin,
DefaultOrganizationMixin,
@@ -21,7 +21,7 @@ from prompt_studio.prompt_studio_v2.models import CustomTool
logger = logging.getLogger(__name__)
-class PromptStudioRegistryModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class PromptStudioRegistryModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
def get_queryset(self) -> QuerySet[Any]:
return super().get_queryset()
diff --git a/backend/tags/models.py b/backend/tags/models.py
index 94c5c8cf5..7f366152b 100644
--- a/backend/tags/models.py
+++ b/backend/tags/models.py
@@ -1,7 +1,7 @@
import uuid
from django.db import models
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
from utils.models.organization_mixin import (
DefaultOrganizationManagerMixin,
DefaultOrganizationMixin,
@@ -9,7 +9,7 @@ from utils.models.organization_mixin import (
from utils.user_context import UserContext
-class TagModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class TagModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
def get_or_create_tags(self, tag_names: list[str]) -> list["Tag"]:
"""Retrieves or creates tags based on a list of tag names.
diff --git a/backend/tool_instance_v2/models.py b/backend/tool_instance_v2/models.py
index 971f1249b..d858da232 100644
--- a/backend/tool_instance_v2/models.py
+++ b/backend/tool_instance_v2/models.py
@@ -3,7 +3,7 @@ import uuid
from account_v2.models import User
from django.db import models
from django.db.models import QuerySet
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
from workflow_manager.workflow_v2.models.workflow import Workflow
TOOL_ID_LENGTH = 64
@@ -11,7 +11,7 @@ TOOL_VERSION_LENGTH = 16
TOOL_STATUS_LENGTH = 32
-class ToolInstanceManager(models.Manager):
+class ToolInstanceManager(BaseModelManager):
def get_instances_for_workflow(self, workflow: uuid.UUID) -> QuerySet["ToolInstance"]:
return self.filter(workflow=workflow)
diff --git a/backend/usage_v2/models.py b/backend/usage_v2/models.py
index 8da3d751b..57ae3d143 100644
--- a/backend/usage_v2/models.py
+++ b/backend/usage_v2/models.py
@@ -1,7 +1,7 @@
import uuid
from django.db import models
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
from utils.models.organization_mixin import (
DefaultOrganizationManagerMixin,
DefaultOrganizationMixin,
@@ -19,7 +19,7 @@ class LLMUsageReason(models.TextChoices):
SUMMARIZE = "summarize", "Summarize"
-class UsageModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class UsageModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
pass
diff --git a/backend/utils/models/base_model.py b/backend/utils/models/base_model.py
index b26f0f67d..c2b288b9e 100644
--- a/backend/utils/models/base_model.py
+++ b/backend/utils/models/base_model.py
@@ -1,9 +1,122 @@
from django.db import models
+from django.utils import timezone
+
+_AUTO_NOW_FIELD = "modified_at"
+
+
+def _with_modified_at(fields):
+ """Return a new list containing ``fields`` plus ``modified_at`` if absent.
+
+ Centralises the "inject modified_at into a partial field list" rule so
+ ``bulk_update``, ``bulk_create`` and ``BaseModel.save`` apply it the same
+ way.
+ """
+ fields = list(fields)
+ if _AUTO_NOW_FIELD not in fields:
+ fields.append(_AUTO_NOW_FIELD)
+ return fields
+
+
+class BaseModelQuerySet(models.QuerySet):
+ """QuerySet that mirrors ``auto_now`` semantics for bulk update paths.
+
+ ``modified_at = models.DateTimeField(auto_now=True)`` only fires on
+ ``Model.save()``. ``QuerySet.update()`` and ``QuerySet.bulk_update()``
+ issue raw SQL and bypass ``save()``, leaving ``modified_at`` at whatever
+ value it had before the bulk path ran (creation time for never-saved
+ rows, the previous save() timestamp for others) — silently drifting the
+ audit trail. This QuerySet patches both paths so callers don't have to
+ remember.
+
+ Callers can still override by passing ``modified_at`` explicitly (or by
+ including ``modified_at`` in the ``fields`` list for ``bulk_update``).
+
+ Note: this is a manager-level convention, not a model-level guarantee.
+ Subclasses that reassign ``objects`` to a plain ``models.Manager``, raw
+ SQL, and migration-time models returned by ``apps.get_model()`` all
+ bypass these overrides.
+ """
+
+ def update(self, **kwargs):
+ kwargs.setdefault(_AUTO_NOW_FIELD, timezone.now())
+ return super().update(**kwargs)
+
+ def bulk_update(self, objs, fields, *args, **kwargs):
+ # Stamp modified_at on each obj only when the caller didn't list it;
+ # materialize objs first because we iterate the sequence twice (once
+ # to stamp, once via super()) and a generator would be exhausted.
+ if _AUTO_NOW_FIELD not in fields:
+ objs = list(objs)
+ now = timezone.now()
+ for obj in objs:
+ obj.modified_at = now
+ fields = _with_modified_at(fields)
+ return super().bulk_update(objs, fields, *args, **kwargs)
+
+ def bulk_create(
+ self, objs, *args, update_conflicts=False, update_fields=None, **kwargs
+ ):
+ # On upsert-on-conflict Django runs an UPDATE with only the listed
+ # fields, which skips auto_now the same way save(update_fields=...)
+ # does. Insert-only bulk_create already handles auto_now itself.
+ if update_conflicts and update_fields:
+ update_fields = _with_modified_at(update_fields)
+ return super().bulk_create(
+ objs,
+ *args,
+ update_conflicts=update_conflicts,
+ update_fields=update_fields,
+ **kwargs,
+ )
+
+
+BaseModelManager = models.Manager.from_queryset(BaseModelQuerySet)
class BaseModel(models.Model):
+ """Abstract base with managed ``created_at`` / ``modified_at`` timestamps.
+
+ Subclasses inherit ``BaseModelManager`` as the default manager, which
+ auto-bumps ``modified_at`` on ``QuerySet.update()``, ``bulk_update()``
+ and upsert-mode ``bulk_create()``. The ``save()`` override below does
+ the same for partial ``save(update_fields=[...])`` calls.
+
+ Subclasses that need a custom manager should compose ``BaseModelManager``
+ (e.g. ``class FooManager(MyMixin, BaseModelManager)``) — otherwise the
+ auto-bump on bulk paths is silently lost.
+ """
+
created_at = models.DateTimeField(auto_now_add=True)
modified_at = models.DateTimeField(auto_now=True)
+ objects = BaseModelManager()
+
class Meta:
abstract = True
+
+ def save(
+ self,
+ force_insert=False,
+ force_update=False,
+ using=None,
+ update_fields=None,
+ **kwargs,
+ ):
+ # Django's save(update_fields=...) only writes the listed columns.
+ # auto_now still updates modified_at on the in-memory instance, but
+ # the new value is never persisted unless modified_at is in
+ # update_fields. Auto-include it so partial saves don't silently drop
+ # the bump. Preserve Django's documented no-op semantics for
+ # update_fields=[] (signals-only save, no column writes).
+ #
+ # Signature mirrors Django's positional order so callers passing
+ # force_insert/force_update positionally still hit this override.
+ if update_fields:
+ update_fields = _with_modified_at(update_fields)
+ return super().save(
+ force_insert=force_insert,
+ force_update=force_update,
+ using=using,
+ update_fields=update_fields,
+ **kwargs,
+ )
diff --git a/backend/utils/models/org_aware_manager.py b/backend/utils/models/org_aware_manager.py
index 3ec6e70bd..ab40303db 100644
--- a/backend/utils/models/org_aware_manager.py
+++ b/backend/utils/models/org_aware_manager.py
@@ -3,15 +3,15 @@
import logging
from django.core.exceptions import ImproperlyConfigured
-from django.db import models
from django.db.utils import OperationalError, ProgrammingError
+from utils.models.base_model import BaseModelManager
from utils.models.org_path_discovery import get_org_path
from utils.user_context import UserContext
logger = logging.getLogger(__name__)
-class OrgAwareManager(models.Manager):
+class OrgAwareManager(BaseModelManager):
"""Manager that auto-discovers FK path to Organization and applies
org filtering to all queries in request context.
diff --git a/backend/workflow_manager/endpoint_v2/models.py b/backend/workflow_manager/endpoint_v2/models.py
index e07737ba7..44742dfc3 100644
--- a/backend/workflow_manager/endpoint_v2/models.py
+++ b/backend/workflow_manager/endpoint_v2/models.py
@@ -2,12 +2,12 @@ import uuid
from connector_v2.models import ConnectorInstance
from django.db import models
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
from utils.user_context import UserContext
from workflow_manager.workflow_v2.models.workflow import Workflow
-class WorkflowEndpointModelManager(models.Manager):
+class WorkflowEndpointModelManager(BaseModelManager):
def get_queryset(self):
# Validating organization
organization = UserContext.get_organization()
diff --git a/backend/workflow_manager/file_execution/models.py b/backend/workflow_manager/file_execution/models.py
index d5106855e..105b3875a 100644
--- a/backend/workflow_manager/file_execution/models.py
+++ b/backend/workflow_manager/file_execution/models.py
@@ -4,7 +4,7 @@ from typing import Any
from django.db import models
from utils.common_utils import CommonUtils
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
from workflow_manager.endpoint_v2.dto import FileHash
from workflow_manager.workflow_v2.enums import ExecutionStatus
@@ -15,7 +15,7 @@ HASH_LENGTH = 64
MIME_TYPE_LENGTH = 128
-class WorkflowFileExecutionManager(models.Manager):
+class WorkflowFileExecutionManager(BaseModelManager):
def get_or_create_file_execution(
self,
workflow_execution: Any,
diff --git a/backend/workflow_manager/internal_views.py b/backend/workflow_manager/internal_views.py
index d11ae5617..c822e5e7b 100644
--- a/backend/workflow_manager/internal_views.py
+++ b/backend/workflow_manager/internal_views.py
@@ -7,7 +7,6 @@ import uuid
from django.db import transaction
from django.shortcuts import get_object_or_404
-from django.utils import timezone
from rest_framework import status, viewsets
from rest_framework.decorators import action
from rest_framework.response import Response
@@ -2009,7 +2008,6 @@ class BatchStatusUpdateAPIView(APIView):
if update.get("execution_time") is not None:
execution.execution_time = update["execution_time"]
- execution.modified_at = timezone.now()
execution.save()
successful_updates.append(
diff --git a/backend/workflow_manager/workflow_v2/file_history_helper.py b/backend/workflow_manager/workflow_v2/file_history_helper.py
index 12b0a9c2c..687f88dfe 100644
--- a/backend/workflow_manager/workflow_v2/file_history_helper.py
+++ b/backend/workflow_manager/workflow_v2/file_history_helper.py
@@ -288,7 +288,6 @@ class FileHistoryHelper:
result=str(result),
metadata=FileHistoryHelper._safe_str(metadata),
error=FileHistoryHelper._safe_str(error),
- modified_at=timezone.now(),
)
# Refresh from DB to get updated values
file_history.refresh_from_db()
diff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.py
index 93623118d..45886bd64 100644
--- a/backend/workflow_manager/workflow_v2/models/execution.py
+++ b/backend/workflow_manager/workflow_v2/models/execution.py
@@ -12,7 +12,7 @@ from usage_v2.constants import UsageKeys
from usage_v2.helper import UsageHelper
from usage_v2.models import Usage
from utils.common_utils import CommonUtils
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
from workflow_manager.execution.dto import ExecutionCache
from workflow_manager.execution.execution_cache_utils import ExecutionCacheUtils
@@ -26,7 +26,7 @@ logger = logging.getLogger(__name__)
EXECUTION_ERROR_LENGTH = 256
-class WorkflowExecutionManager(models.Manager):
+class WorkflowExecutionManager(BaseModelManager):
"""Custom manager for WorkflowExecution model to handle user-specific filtering."""
def for_user(self, user) -> QuerySet:
diff --git a/backend/workflow_manager/workflow_v2/models/workflow.py b/backend/workflow_manager/workflow_v2/models/workflow.py
index dd945fc7f..0029f9599 100644
--- a/backend/workflow_manager/workflow_v2/models/workflow.py
+++ b/backend/workflow_manager/workflow_v2/models/workflow.py
@@ -4,7 +4,7 @@ from account_v2.models import User
from django.conf import settings
from django.core.validators import MinValueValidator
from django.db import models
-from utils.models.base_model import BaseModel
+from utils.models.base_model import BaseModel, BaseModelManager
from utils.models.organization_mixin import (
DefaultOrganizationManagerMixin,
DefaultOrganizationMixin,
@@ -15,7 +15,7 @@ DESCRIPTION_FIELD_LENGTH = 490
WORKFLOW_NAME_SIZE = 128
-class WorkflowModelManager(DefaultOrganizationManagerMixin, models.Manager):
+class WorkflowModelManager(DefaultOrganizationManagerMixin, BaseModelManager):
def for_user(self, user):
"""Filter workflows that the user can access:
- Workflows created by the user
diff --git a/backend/workflow_manager/workflow_v2/views.py b/backend/workflow_manager/workflow_v2/views.py
index dc428d948..629b52bb2 100644
--- a/backend/workflow_manager/workflow_v2/views.py
+++ b/backend/workflow_manager/workflow_v2/views.py
@@ -6,7 +6,6 @@ from django.conf import settings
from django.db import transaction
from django.db.models.query import QuerySet
from django.shortcuts import get_object_or_404
-from django.utils import timezone
from django.views.decorators.csrf import csrf_exempt
from permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg
from pipeline_v2.models import Pipeline
@@ -528,7 +527,6 @@ class WorkflowExecutionInternalViewSet(viewsets.ReadOnlyModelViewSet):
if validated_data.get("execution_time") is not None:
execution.execution_time = validated_data["execution_time"]
- execution.modified_at = timezone.now()
execution.save()
logger.info(