Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat(asyncify): migrate jobs (#2420)
cthomasc071e079dced
diff --git a/letta/server/rest_api/routers/v1/jobs.py b/letta/server/rest_api/routers/v1/jobs.pyindex 9c0cba4e5..d0c7a2e64 100644--- a/letta/server/rest_api/routers/v1/jobs.py+++ b/letta/server/rest_api/routers/v1/jobs.py@@ -12,7 +12,7 @@ router = APIRouter(prefix="/jobs", tags=["jobs"])@router.get("/", response_model=List[Job], operation_id="list_jobs")-def list_jobs(+async def list_jobs(server: "SyncServer" = Depends(get_letta_server),source_id: Optional[str] = Query(None, description="Only list jobs associated with the source."),actor_id: Optional[str] = Header(None, alias="user_id"), # Extract user_id from header, default to None if not present@@ -20,10 +20,10 @@ def list_jobs("""List all jobs."""- actor = server.user_manager.get_user_or_default(user_id=actor_id)+ actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)# TODO: add filtering by status- jobs = server.job_manager.list_jobs(actor=actor)+ jobs = await server.job_manager.list_jobs_async(actor=actor)if source_id:# can't be in the ORM since we have source_id stored in the metadata@@ -46,7 +46,7 @@ async def list_active_jobs(@router.get("/{job_id}", response_model=Job, operation_id="retrieve_job")-def retrieve_job(+async def retrieve_job(job_id: str,actor_id: Optional[str] = Header(None, alias="user_id"),server: "SyncServer" = Depends(get_letta_server),@@ -54,16 +54,16 @@ def retrieve_job("""Get the status of a job."""- actor = server.user_manager.get_user_or_default(user_id=actor_id)+ actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)try:- return server.job_manager.get_job_by_id(job_id=job_id, actor=actor)+ return await server.job_manager.get_job_by_id_async(job_id=job_id, actor=actor)except NoResultFound:raise HTTPException(status_code=404, detail="Job not found")@router.delete("/{job_id}", response_model=Job, operation_id="delete_job")-def delete_job(+async def delete_job(job_id: str,actor_id: Optional[str] = Header(None, alias="user_id"),server: "SyncServer" = Depends(get_letta_server),@@ -71,10 +71,10 @@ def delete_job("""Delete a job by its job_id."""- actor = server.user_manager.get_user_or_default(user_id=actor_id)+ actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)try:- job = server.job_manager.delete_job_by_id(job_id=job_id, actor=actor)+ job = await server.job_manager.delete_job_by_id_async(job_id=job_id, actor=actor)return jobexcept NoResultFound:raise HTTPException(status_code=404, detail="Job not found")diff --git a/letta/server/rest_api/routers/v1/runs.py b/letta/server/rest_api/routers/v1/runs.pyindex 8a8793a39..ac0476e4a 100644--- a/letta/server/rest_api/routers/v1/runs.py+++ b/letta/server/rest_api/routers/v1/runs.py@@ -199,7 +199,7 @@ async def list_run_steps(@router.delete("/{run_id}", response_model=Run, operation_id="delete_run")-def delete_run(+async def delete_run(run_id: str,actor_id: Optional[str] = Header(None, alias="user_id"),server: "SyncServer" = Depends(get_letta_server),@@ -207,10 +207,10 @@ def delete_run("""Delete a run by its run_id."""- actor = server.user_manager.get_user_or_default(user_id=actor_id)+ actor = await server.user_manager.get_actor_or_default_async(actor_id=actor_id)try:- job = server.job_manager.delete_job_by_id(job_id=run_id, actor=actor)+ job = await server.job_manager.delete_job_by_id_async(job_id=run_id, actor=actor)return Run.from_job(job)except NoResultFound:raise HTTPException(status_code=404, detail="Run not found")diff --git a/letta/services/job_manager.py b/letta/services/job_manager.pyindex 3cd1ee035..1cdc5c58f 100644--- a/letta/services/job_manager.py+++ b/letta/services/job_manager.py@@ -197,6 +197,15 @@ class JobManager:job.hard_delete(db_session=session, actor=actor)return job.to_pydantic()+ @enforce_types+ @trace_method+ async def delete_job_by_id_async(self, job_id: str, actor: PydanticUser) -> PydanticJob:+ """Delete a job by its ID."""+ async with db_registry.async_session() as session:+ job = await self._verify_job_access_async(session=session, job_id=job_id, actor=actor)+ await job.hard_delete_async(db_session=session, actor=actor)+ return job.to_pydantic()+@enforce_types@trace_methoddef get_job_messages(@@ -599,25 +608,23 @@ class JobManager:async def _dispatch_callback_async(self, session, job: JobModel) -> None:"""- POST a standard JSON payload to job.callback_url- and record timestamp + HTTP status asynchronously.+ POST a standard JSON payload to job.callback_url and record timestamp + HTTP status asynchronously."""-payload = {"job_id": job.id,"status": job.status,- "completed_at": job.completed_at.isoformat(),+ "completed_at": job.completed_at.isoformat() if job.completed_at else None,}+try:import httpxasync with httpx.AsyncClient() as client:resp = await client.post(job.callback_url, json=payload, timeout=5.0)- job.callback_sent_at = get_utc_time()+ # Ensure timestamp is timezone-naive for DB compatibility+ job.callback_sent_at = get_utc_time().replace(tzinfo=None)job.callback_status_code = resp.status_code-except Exception:- return-- session.add(job)- await session.commit()+ # Silently fail on callback errors - job updates should still succeed+ # In production, this would include proper error logging+ passdiff --git a/tests/test_managers.py b/tests/test_managers.pyindex 8a7d57df4..e033b9948 100644--- a/tests/test_managers.py+++ b/tests/test_managers.py@@ -4245,14 +4245,15 @@ def test_get_sandbox_env_var_by_key(server: SyncServer, sandbox_env_var_fixture,# ======================================================================================================================-def test_create_job(server: SyncServer, default_user):+async def test_create_job(server: SyncServer, default_user, event_loop):"""Test creating a job."""job_data = PydanticJob(status=JobStatus.created,metadata={"type": "test"},)- created_job = server.job_manager.create_job(job_data, actor=default_user)+ created_job = await server.job_manager.create_job_async(pydantic_job=job_data, actor=default_user)# Assertions to ensure the created job matches the expected valuesassert created_job.user_id == default_user.id@@ -4260,17 +4261,18 @@ def test_create_job(server: SyncServer, default_user):assert created_job.metadata == {"type": "test"}-def test_get_job_by_id(server: SyncServer, default_user):+async def test_get_job_by_id(server: SyncServer, default_user, event_loop):"""Test fetching a job by ID."""# Create a jobjob_data = PydanticJob(status=JobStatus.created,metadata={"type": "test"},)- created_job = server.job_manager.create_job(job_data, actor=default_user)+ created_job = await server.job_manager.create_job_async(pydantic_job=job_data, actor=default_user)# Fetch the job by ID- fetched_job = server.job_manager.get_job_by_id(created_job.id, actor=default_user)+ fetched_job = await server.job_manager.get_job_by_id_async(created_job.id, actor=default_user)# Assertions to ensure the fetched job matches the created jobassert fetched_job.id == created_job.id@@ -4278,7 +4280,8 @@ def test_get_job_by_id(server: SyncServer, default_user):assert fetched_job.metadata == {"type": "test"}-def test_list_jobs(server: SyncServer, default_user):+async def test_list_jobs(server: SyncServer, default_user, event_loop):"""Test listing jobs."""# Create multiple jobsfor i in range(3):@@ -4286,10 +4289,10 @@ def test_list_jobs(server: SyncServer, default_user):status=JobStatus.created,metadata={"type": f"test-{i}"},)- server.job_manager.create_job(job_data, actor=default_user)+ await server.job_manager.create_job_async(pydantic_job=job_data, actor=default_user)# List jobs- jobs = server.job_manager.list_jobs(actor=default_user)+ jobs = await server.job_manager.list_jobs_async(actor=default_user)# Assertions to check that the created jobs are listedassert len(jobs) == 3@@ -4297,19 +4300,20 @@ def test_list_jobs(server: SyncServer, default_user):assert all(job.metadata["type"].startswith("test") for job in jobs)-def test_update_job_by_id(server: SyncServer, default_user):+async def test_update_job_by_id(server: SyncServer, default_user, event_loop):"""Test updating a job by its ID."""# Create a jobjob_data = PydanticJob(status=JobStatus.created,metadata={"type": "test"},)- created_job = server.job_manager.create_job(job_data, actor=default_user)+ created_job = await server.job_manager.create_job_async(pydantic_job=job_data, actor=default_user)assert created_job.metadata == {"type": "test"}# Update the jobupdate_data = JobUpdate(status=JobStatus.completed, metadata={"type": "updated"})- updated_job = server.job_manager.update_job_by_id(created_job.id, update_data, actor=default_user)+ updated_job = await server.job_manager.update_job_by_id_async(created_job.id, update_data, actor=default_user)# Assertions to ensure the job was updatedassert updated_job.status == JobStatus.completed@@ -4317,56 +4321,61 @@ def test_update_job_by_id(server: SyncServer, default_user):assert updated_job.completed_at is not None-def test_delete_job_by_id(server: SyncServer, default_user):+async def test_delete_job_by_id(server: SyncServer, default_user, event_loop):"""Test deleting a job by its ID."""# Create a jobjob_data = PydanticJob(status=JobStatus.created,metadata={"type": "test"},)- created_job = server.job_manager.create_job(job_data, actor=default_user)+ created_job = await server.job_manager.create_job_async(pydantic_job=job_data, actor=default_user)# Delete the job- server.job_manager.delete_job_by_id(created_job.id, actor=default_user)+ await server.job_manager.delete_job_by_id_async(created_job.id, actor=default_user)# List jobs to ensure the job was deleted- jobs = server.job_manager.list_jobs(actor=default_user)+ jobs = await server.job_manager.list_jobs_async(actor=default_user)assert len(jobs) == 0-def test_update_job_auto_complete(server: SyncServer, default_user):+async def test_update_job_auto_complete(server: SyncServer, default_user, event_loop):"""Test that updating a job's status to 'completed' automatically sets completed_at."""# Create a jobjob_data = PydanticJob(status=JobStatus.created,metadata={"type": "test"},)- created_job = server.job_manager.create_job(job_data, actor=default_user)+ created_job = await server.job_manager.create_job_async(pydantic_job=job_data, actor=default_user)# Update the job's status to 'completed'update_data = JobUpdate(status=JobStatus.completed)- updated_job = server.job_manager.update_job_by_id(created_job.id, update_data, actor=default_user)+ updated_job = await server.job_manager.update_job_by_id_async(created_job.id, update_data, actor=default_user)# Assertions to check that completed_at was setassert updated_job.status == JobStatus.completedassert updated_job.completed_at is not None-def test_get_job_not_found(server: SyncServer, default_user):+async def test_get_job_not_found(server: SyncServer, default_user, event_loop):"""Test fetching a non-existent job."""non_existent_job_id = "nonexistent-id"with pytest.raises(NoResultFound):- server.job_manager.get_job_by_id(non_existent_job_id, actor=default_user)+ await server.job_manager.get_job_by_id_async(non_existent_job_id, actor=default_user)-def test_delete_job_not_found(server: SyncServer, default_user):+async def test_delete_job_not_found(server: SyncServer, default_user, event_loop):"""Test deleting a non-existent job."""non_existent_job_id = "nonexistent-id"with pytest.raises(NoResultFound):- server.job_manager.delete_job_by_id(non_existent_job_id, actor=default_user)+ await server.job_manager.delete_job_by_id_async(non_existent_job_id, actor=default_user)-def test_list_jobs_pagination(server: SyncServer, default_user):+async def test_list_jobs_pagination(server: SyncServer, default_user, event_loop):"""Test listing jobs with pagination."""# Create multiple jobsfor i in range(10):@@ -4374,19 +4383,19 @@ def test_list_jobs_pagination(server: SyncServer, default_user):status=JobStatus.created,metadata={"type": f"test-{i}"},)- server.job_manager.create_job(job_data, actor=default_user)+ await server.job_manager.create_job_async(pydantic_job=job_data, actor=default_user)# List jobs with a limit- jobs = server.job_manager.list_jobs(actor=default_user, limit=5)+ jobs = await server.job_manager.list_jobs_async(actor=default_user, limit=5)assert len(jobs) == 5assert all(job.user_id == default_user.id for job in jobs)# Test cursor-based pagination- first_page = server.job_manager.list_jobs(actor=default_user, limit=3, ascending=True) # [J0, J1, J2]+ first_page = await server.job_manager.list_jobs_async(actor=default_user, limit=3, ascending=True) # [J0, J1, J2]assert len(first_page) == 3assert first_page[0].created_at <= first_page[1].created_at <= first_page[2].created_at- last_page = server.job_manager.list_jobs(actor=default_user, limit=3, ascending=False) # [J9, J8, J7]+ last_page = await server.job_manager.list_jobs_async(actor=default_user, limit=3, ascending=False) # [J9, J8, J7]assert len(last_page) == 3assert last_page[0].created_at >= last_page[1].created_at >= last_page[2].created_atfirst_page_ids = set(job.id for job in first_page)@@ -4394,7 +4403,7 @@ def test_list_jobs_pagination(server: SyncServer, default_user):assert first_page_ids.isdisjoint(last_page_ids)# Test middle page using both before and after- middle_page = server.job_manager.list_jobs(+ middle_page = await server.job_manager.list_jobs_async(actor=default_user, before=last_page[-1].id, after=first_page[-1].id, ascending=True) # [J3, J4, J5, J6]assert len(middle_page) == 4 # Should include jobs between first and second page@@ -4402,7 +4411,7 @@ def test_list_jobs_pagination(server: SyncServer, default_user):assert all(job.id not in head_tail_jobs for job in middle_page)# Test descending order- middle_page_desc = server.job_manager.list_jobs(+ middle_page_desc = await server.job_manager.list_jobs_async(actor=default_user, before=last_page[-1].id, after=first_page[-1].id, ascending=False) # [J6, J5, J4, J3]assert len(middle_page_desc) == 4@@ -4413,13 +4422,14 @@ def test_list_jobs_pagination(server: SyncServer, default_user):# BONUSjob_7 = last_page[-1].id- earliest_jobs = server.job_manager.list_jobs(actor=default_user, ascending=False, before=job_7)+ earliest_jobs = await server.job_manager.list_jobs_async(actor=default_user, ascending=False, before=job_7)assert len(earliest_jobs) == 7assert all(j.id not in last_page_ids for j in earliest_jobs)assert all(earliest_jobs[i].created_at >= earliest_jobs[i + 1].created_at for i in range(len(earliest_jobs) - 1))-def test_list_jobs_by_status(server: SyncServer, default_user):+async def test_list_jobs_by_status(server: SyncServer, default_user, event_loop):"""Test listing jobs filtered by status."""# Create multiple jobs with different statusesjob_data_created = PydanticJob(@@ -4435,14 +4445,14 @@ def test_list_jobs_by_status(server: SyncServer, default_user):metadata={"type": "test-completed"},)- server.job_manager.create_job(job_data_created, actor=default_user)- server.job_manager.create_job(job_data_in_progress, actor=default_user)- server.job_manager.create_job(job_data_completed, actor=default_user)+ await server.job_manager.create_job_async(pydantic_job=job_data_created, actor=default_user)+ await server.job_manager.create_job_async(pydantic_job=job_data_in_progress, actor=default_user)+ await server.job_manager.create_job_async(pydantic_job=job_data_completed, actor=default_user)# List jobs filtered by status- created_jobs = server.job_manager.list_jobs(actor=default_user, statuses=[JobStatus.created])- in_progress_jobs = server.job_manager.list_jobs(actor=default_user, statuses=[JobStatus.running])- completed_jobs = server.job_manager.list_jobs(actor=default_user, statuses=[JobStatus.completed])+ created_jobs = await server.job_manager.list_jobs_async(actor=default_user, statuses=[JobStatus.created])+ in_progress_jobs = await server.job_manager.list_jobs_async(actor=default_user, statuses=[JobStatus.running])+ completed_jobs = await server.job_manager.list_jobs_async(actor=default_user, statuses=[JobStatus.completed])# Assertionsassert len(created_jobs) == 1@@ -4455,7 +4465,8 @@ def test_list_jobs_by_status(server: SyncServer, default_user):assert completed_jobs[0].metadata["type"] == job_data_completed.metadata["type"]-def test_list_jobs_filter_by_type(server: SyncServer, default_user, default_job):+async def test_list_jobs_filter_by_type(server: SyncServer, default_user, default_job, event_loop):"""Test that list_jobs correctly filters by job_type."""# Create a run jobrun_pydantic = PydanticJob(@@ -4463,48 +4474,62 @@ def test_list_jobs_filter_by_type(server: SyncServer, default_user, default_job)status=JobStatus.pending,job_type=JobType.RUN,)- run = server.job_manager.create_job(pydantic_job=run_pydantic, actor=default_user)+ run = await server.job_manager.create_job_async(pydantic_job=run_pydantic, actor=default_user)# List only regular jobs- jobs = server.job_manager.list_jobs(actor=default_user)+ jobs = await server.job_manager.list_jobs_async(actor=default_user)assert len(jobs) == 1assert jobs[0].id == default_job.id# List only run jobs- jobs = server.job_manager.list_jobs(actor=default_user, job_type=JobType.RUN)+ jobs = await server.job_manager.list_jobs_async(actor=default_user, job_type=JobType.RUN)assert len(jobs) == 1assert jobs[0].id == run.id-def test_e2e_job_callback(monkeypatch, server: SyncServer, default_user):+async def test_e2e_job_callback(monkeypatch, server: SyncServer, default_user):+ """Test that job callbacks are properly dispatched when a job is completed."""captured = {}- def fake_post(url, json, timeout):+ # Create a simple mock for the async HTTP client+ class MockAsyncResponse:+ status_code = 202++ async def mock_post(url, json, timeout):captured["url"] = urlcaptured["json"] = json+ return MockAsyncResponse()++ class MockAsyncClient:+ async def __aenter__(self):+ return self- class FakeResponse:- status_code = 202+ async def __aexit__(self, *args):+ pass- return FakeResponse()+ async def post(self, url, json, timeout):+ return await mock_post(url, json, timeout)- monkeypatch.setattr(httpx, "post", fake_post)+ # Patch the AsyncClient+ monkeypatch.setattr(httpx, "AsyncClient", MockAsyncClient)job_in = PydanticJob(status=JobStatus.created, metadata={"foo": "bar"}, callback_url="http://example.test/webhook/jobs")- created = server.job_manager.create_job(job_in, actor=default_user)+ created = await server.job_manager.create_job_async(pydantic_job=job_in, actor=default_user)assert created.callback_url == "http://example.test/webhook/jobs"+ # Update the job status to completed, which should trigger the callbackupdate = JobUpdate(status=JobStatus.completed)- updated = server.job_manager.update_job_by_id(created.id, update, actor=default_user)+ updated = await server.job_manager.update_job_by_id_async(created.id, update, actor=default_user)- assert captured["url"] == created.callback_url- assert captured["json"]["job_id"] == created.id- assert captured["json"]["status"] == JobStatus.completed.value+ # Verify the callback was triggered with the correct parameters+ assert captured["url"] == created.callback_url, "Callback URL doesn't match"+ assert captured["json"]["job_id"] == created.id, "Job ID in callback doesn't match"+ assert captured["json"]["status"] == JobStatus.completed.value, "Job status in callback doesn't match"- # Normalize the received completed_at to compare properly+ # Verify the completed_at timestamp is reasonableactual_dt = datetime.fromisoformat(captured["json"]["completed_at"]).replace(tzinfo=None)- expected_dt = updated.completed_at.replace(tzinfo=None)- assert actual_dt == expected_dt+ assert abs((actual_dt - updated.completed_at).total_seconds()) < 1, "Timestamp difference is too large"assert isinstance(updated.callback_sent_at, datetime)assert updated.callback_status_code == 202