Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

14.0 KB
Raw Blame
set,scenario,session,variant,raw_check_count,mini_raw_passed,original_luna_raw_passed,migrated_luna_raw_passed,adjudicated_check_count,mini_passed,original_luna_passed,migrated_luna_passed,mini_failed_checks,original_luna_failed_checks,migrated_luna_failed_checks
primary,mem-abstain-ambiguous-reference,1,facts_only,3,3,3,3,3,3,3,3,,,
primary,mem-abstain-ambiguous-reference,1,memory_off,3,0,0,0,3,0,0,0,brightpath|brightcafe|ambiguity,brightpath|brightcafe|ambiguity,brightpath|brightcafe|ambiguity
primary,mem-abstain-ambiguous-reference,1,normal,3,3,3,3,3,3,3,3,,,
primary,mem-abstain-no-fabricated-preferences,1,facts_only,1,1,1,1,1,1,1,1,,,
primary,mem-abstain-no-fabricated-preferences,1,memory_off,1,1,1,1,1,1,1,1,,,
primary,mem-abstain-no-fabricated-preferences,1,normal,1,1,1,1,1,1,1,1,,,
primary,mem-compositional-board-prep,2,facts_only,3,2,2,3,3,2,2,3,year_end,year_end,
primary,mem-compositional-board-prep,2,memory_off,3,0,0,0,3,0,0,0,year_end|lead|duration,year_end|lead|duration,year_end|lead|duration
primary,mem-compositional-board-prep,2,normal,3,3,3,3,3,3,3,3,,,
primary,mem-continuation-interrupted-task,1,facts_only,4,0,0,0,4,0,0,0,david|novatech|lisa|brightpath,david|novatech|lisa|brightpath,david|novatech|lisa|brightpath
primary,mem-continuation-interrupted-task,1,memory_off,4,0,0,0,4,0,0,0,david|novatech|lisa|brightpath,david|novatech|lisa|brightpath,david|novatech|lisa|brightpath
primary,mem-continuation-interrupted-task,1,normal,4,4,4,4,4,4,4,4,,,
primary,mem-continuation-project-state,2,facts_only,7,7,4,7,7,7,4,7,,notion|office|total,
primary,mem-continuation-project-state,2,memory_off,7,0,0,0,7,0,0,0,adobe|aws|zoom|notion|office|total|largest,adobe|aws|zoom|notion|office|total|largest,adobe|aws|zoom|notion|office|total|largest
primary,mem-continuation-project-state,2,normal,7,7,5,7,7,7,5,7,,total|largest,
primary,mem-crossdomain-business-identity,1,facts_only,6,4,4,6,6,4,4,6,ein|title,state|ein,
primary,mem-crossdomain-business-identity,1,memory_off,6,0,0,0,6,0,0,0,company|entity|state|ein|signer|title,company|entity|state|ein|signer|title,company|entity|state|ein|signer|title
primary,mem-crossdomain-business-identity,1,normal,6,6,6,6,6,6,6,6,,,
primary,mem-crossdomain-customer-allergy-with-negative,1,facts_only,2,2,2,2,2,2,2,2,,,
primary,mem-crossdomain-customer-allergy-with-negative,2,facts_only,1,1,1,1,1,1,1,1,,,
primary,mem-crossdomain-customer-allergy-with-negative,1,memory_off,2,0,0,0,2,0,0,0,peanut|gluten,peanut|gluten,peanut|gluten
primary,mem-crossdomain-customer-allergy-with-negative,2,memory_off,1,1,1,1,1,1,1,1,,,
primary,mem-crossdomain-customer-allergy-with-negative,1,normal,2,2,2,2,2,2,2,2,,,
primary,mem-crossdomain-customer-allergy-with-negative,2,normal,1,1,1,1,1,1,1,1,,,
primary,mem-crossdomain-notion-rate-limit,1,facts_only,2,1,2,1,1,1,1,1,,,
primary,mem-crossdomain-notion-rate-limit,1,memory_off,2,0,0,0,1,0,0,0,limit,limit,limit
primary,mem-crossdomain-notion-rate-limit,1,normal,2,2,2,1,1,1,1,1,,,
primary,mem-crossdomain-pricing-structure-with-negative,1,facts_only,3,0,0,0,3,0,0,0,wholesale|discount|net,wholesale|discount|net,wholesale|discount|net
primary,mem-crossdomain-pricing-structure-with-negative,2,facts_only,3,1,1,1,3,1,1,1,retail|no_discount,retail|no_discount,retail|no_discount
primary,mem-crossdomain-pricing-structure-with-negative,1,memory_off,3,0,0,0,3,0,0,0,wholesale|discount|net,wholesale|discount|net,wholesale|discount|net
primary,mem-crossdomain-pricing-structure-with-negative,2,memory_off,3,1,1,1,3,1,1,1,retail|no_discount,retail|no_discount,retail|no_discount
primary,mem-crossdomain-pricing-structure-with-negative,1,normal,3,1,0,0,3,1,0,0,discount|net,wholesale|discount|net,wholesale|discount|net
primary,mem-crossdomain-pricing-structure-with-negative,2,normal,3,1,1,1,3,1,1,1,retail|no_discount,retail|no_discount,retail|no_discount
primary,mem-crossdomain-shipping-schedule-reasoning,1,facts_only,4,4,0,3,4,4,0,3,,city|ship|arrival_friday|arrival_monday,arrival_monday
primary,mem-crossdomain-shipping-schedule-reasoning,1,memory_off,4,0,0,0,4,0,0,0,city|ship|arrival_friday|arrival_monday,city|ship|arrival_friday|arrival_monday,city|ship|arrival_friday|arrival_monday
primary,mem-crossdomain-shipping-schedule-reasoning,1,normal,4,4,4,4,4,4,4,4,,,
primary,mem-distill-authed-http-image-gen,1,facts_only,8,2,2,8,8,2,2,8,method|endpoint|auth|key_name|size|response,tool|method|endpoint|auth|size|response,
primary,mem-distill-authed-http-image-gen,1,memory_off,8,0,0,0,8,0,0,0,tool|method|endpoint|auth|key_name|model|size|response,tool|method|endpoint|auth|key_name|model|size|response,tool|method|endpoint|auth|key_name|model|size|response
primary,mem-distill-authed-http-image-gen,1,normal,8,8,8,8,8,8,8,8,,,
primary,mem-distill-implicit-tool-preferences,1,facts_only,4,4,4,4,4,4,4,4,,,
primary,mem-distill-implicit-tool-preferences,1,memory_off,4,0,0,0,4,0,0,0,scrape|storage|location|email,scrape|storage|location|email,scrape|storage|location|email
primary,mem-distill-implicit-tool-preferences,1,normal,4,4,4,4,4,4,4,4,,,
primary,mem-distill-lead-cleaning-procedure,1,facts_only,5,4,0,5,5,4,0,5,names,dedup_email|phones|names|drop_email|flag_company,
primary,mem-distill-lead-cleaning-procedure,1,memory_off,5,0,0,0,5,0,0,0,dedup_email|phones|names|drop_email|flag_company,dedup_email|phones|names|drop_email|flag_company,dedup_email|phones|names|drop_email|flag_company
primary,mem-distill-lead-cleaning-procedure,1,normal,5,5,5,5,5,5,5,5,,,
primary,mem-distill-onboarding-workflow,1,facts_only,7,7,7,7,7,7,7,7,,,
primary,mem-distill-onboarding-workflow,1,memory_off,7,0,0,0,7,0,0,0,crm|notion|database|tasks|count|spacing|email,crm|notion|database|tasks|count|spacing|email,crm|notion|database|tasks|count|spacing|email
primary,mem-distill-onboarding-workflow,1,normal,7,7,7,7,7,7,7,7,,,
primary,mem-distill-weekly-report-format,1,facts_only,6,4,0,6,6,4,0,6,sections|tone,sections|tone|source|recipient1|recipient2|subject,
primary,mem-distill-weekly-report-format,1,memory_off,6,0,0,0,6,0,0,0,sections|tone|source|recipient1|recipient2|subject,sections|tone|source|recipient1|recipient2|subject,sections|tone|source|recipient1|recipient2|subject
primary,mem-distill-weekly-report-format,1,normal,6,6,6,6,6,6,6,6,,,
primary,mem-hygiene-bounded-time,1,facts_only,1,0,0,0,1,0,0,0,active,active,active
primary,mem-hygiene-bounded-time,2,facts_only,1,0,0,0,1,0,0,0,expired,expired,expired
primary,mem-hygiene-bounded-time,1,memory_off,1,0,0,0,1,0,0,0,active,active,active
primary,mem-hygiene-bounded-time,2,memory_off,1,0,0,0,1,0,0,0,expired,expired,expired
primary,mem-hygiene-bounded-time,1,normal,1,0,0,0,1,0,0,0,active,active,active
primary,mem-hygiene-bounded-time,2,normal,1,1,1,1,1,1,1,1,,,
primary,mem-hygiene-temporary-status,1,facts_only,2,2,2,2,2,2,2,2,,,
primary,mem-hygiene-temporary-status,2,facts_only,2,2,2,2,2,2,2,2,,,
primary,mem-hygiene-temporary-status,1,memory_off,2,0,0,0,2,0,0,0,stockout|return,stockout|return,stockout|return
primary,mem-hygiene-temporary-status,2,memory_off,2,1,1,2,2,1,1,2,expired,expired,
primary,mem-hygiene-temporary-status,1,normal,2,2,2,2,2,2,2,2,,,
primary,mem-hygiene-temporary-status,2,normal,2,2,2,2,2,2,2,2,,,
primary,mem-introspection-gaps,1,facts_only,2,2,2,2,2,2,2,2,,,
primary,mem-introspection-gaps,1,memory_off,2,2,2,2,2,2,2,2,,,
primary,mem-introspection-gaps,1,normal,2,2,2,2,2,2,2,2,,,
primary,mem-introspection-what-do-you-know,2,facts_only,14,11,8,14,13,10,7,13,fiscal|assistant|cycle,structure|fiscal|growth|standard|wholesale|threshold,
primary,mem-introspection-what-do-you-know,2,memory_off,14,0,0,0,13,0,0,0,structure|state|fiscal|assistant|crm|shipping|days|competitor|cycle|growth|standard|wholesale|threshold,structure|state|fiscal|assistant|crm|shipping|days|competitor|cycle|growth|standard|wholesale|threshold,structure|state|fiscal|assistant|crm|shipping|days|competitor|cycle|growth|standard|wholesale|threshold
primary,mem-introspection-what-do-you-know,2,normal,14,14,14,14,13,13,13,13,,,
primary,mem-longtail-lawyer-recall,4,facts_only,3,2,2,3,3,2,2,3,email,email,
primary,mem-longtail-lawyer-recall,4,memory_off,3,0,0,0,3,0,0,0,name|firm|email,name|firm|email,name|firm|email
primary,mem-longtail-lawyer-recall,4,normal,3,3,3,3,3,3,3,3,,,
primary,mem-negative-forget-on-request,2,facts_only,2,0,0,0,excluded,,,,,,
primary,mem-negative-forget-on-request,2,memory_off,2,2,2,2,excluded,,,,,,
primary,mem-negative-forget-on-request,2,normal,2,2,2,2,excluded,,,,,,
primary,mem-negative-one-off-qualifier,1,facts_only,1,1,1,1,1,1,1,1,,,
primary,mem-negative-one-off-qualifier,1,memory_off,1,1,1,1,1,1,1,1,,,
primary,mem-negative-one-off-qualifier,1,normal,1,1,1,1,1,1,1,1,,,
primary,mem-procupdate-additive,2,facts_only,7,7,7,7,7,7,7,7,,,
primary,mem-procupdate-additive,2,memory_off,7,0,0,0,7,0,0,0,crm|database|tasks|count|spacing|email|cc,crm|database|tasks|count|spacing|email|cc,crm|database|tasks|count|spacing|email|cc
primary,mem-procupdate-additive,2,normal,7,7,7,7,7,7,7,7,,,
primary,mem-procupdate-clean-replacement,2,facts_only,9,9,1,9,9,9,1,9,,crm|database|base|table|tasks|count|spacing|email,
primary,mem-procupdate-clean-replacement,2,memory_off,9,1,1,1,9,1,1,1,crm|database|base|table|tasks|count|spacing|email,crm|database|base|table|tasks|count|spacing|email,crm|database|base|table|tasks|count|spacing|email
primary,mem-procupdate-clean-replacement,2,normal,9,9,9,9,9,9,9,9,,,
primary,mem-retention-basic-identity,1,facts_only,3,2,2,3,3,2,2,3,invoices,assistant,
primary,mem-retention-basic-identity,1,memory_off,3,0,0,0,3,0,0,0,assistant|crm|invoices,assistant|crm|invoices,assistant|crm|invoices
primary,mem-retention-basic-identity,1,normal,3,3,3,3,3,3,3,3,,,
primary,mem-retention-incidental-facts,1,facts_only,6,6,3,6,6,6,3,6,,fiscal_year|sales_cycle|font,
primary,mem-retention-incidental-facts,1,memory_off,6,0,0,0,6,0,0,0,fiscal_year|competitor|warehouse|billing|sales_cycle|font,fiscal_year|competitor|warehouse|billing|sales_cycle|font,fiscal_year|competitor|warehouse|billing|sales_cycle|font
primary,mem-retention-incidental-facts,1,normal,6,6,6,6,6,6,6,6,,,
primary,mem-rigidity-email-tone-override,2,facts_only,4,0,4,4,4,0,4,4,length|tone|opener|forbidden_0,,
primary,mem-rigidity-email-tone-override,2,memory_off,4,1,1,1,4,1,1,1,length|tone|opener,length|tone|opener,length|tone|opener
primary,mem-rigidity-email-tone-override,2,normal,4,4,4,4,4,4,4,4,,,
primary,mem-rigidity-pricing-update,2,facts_only,2,2,2,2,2,2,2,2,,,
primary,mem-rigidity-pricing-update,2,memory_off,2,1,1,1,2,1,1,1,price,price,price
primary,mem-rigidity-pricing-update,2,normal,2,2,2,2,2,2,2,2,,,
primary,mem-rigidity-tool-migration,2,facts_only,4,4,4,4,4,4,4,4,,,
primary,mem-rigidity-tool-migration,2,memory_off,4,1,1,1,4,1,1,1,crm|base|table,crm|base|table,crm|base|table
primary,mem-rigidity-tool-migration,2,normal,4,4,4,4,4,4,4,4,,,
primary,mem-temporal-deprecated-procedure,2,facts_only,6,6,5,6,6,6,5,6,,review,
primary,mem-temporal-deprecated-procedure,2,memory_off,6,0,0,0,6,0,0,0,reply|database|log|base|review|no_discord,reply|database|log|base|review|no_discord,reply|database|log|base|review|no_discord
primary,mem-temporal-deprecated-procedure,2,normal,6,6,6,6,6,6,6,6,,,
primary,mem-temporal-stale-team-member,2,facts_only,3,2,3,3,3,2,3,3,email,,
primary,mem-temporal-stale-team-member,2,memory_off,3,1,1,1,3,1,1,1,name|email,name|email,name|email
primary,mem-temporal-stale-team-member,2,normal,3,3,3,3,3,3,3,3,,,
controls,heldout-conditional-policy,1,facts_only,5,5,,5,5,5,,5,,,
controls,heldout-conditional-policy,1,memory_off,5,0,,0,5,0,,0,large_price|discount|small_price|small_no_discount|shipping,,large_price|discount|small_price|small_no_discount|shipping
controls,heldout-conditional-policy,1,normal,5,5,,5,5,5,,5,,,
controls,heldout-incidental-units,1,facts_only,6,6,,6,6,6,,6,,,
controls,heldout-incidental-units,1,memory_off,6,0,,0,6,0,,0,payment|year|font|city|days|minimum,,payment|year|font|city|days|minimum
controls,heldout-incidental-units,1,normal,6,6,,6,6,6,,6,,,
controls,heldout-one-off-exception,1,facts_only,5,4,,5,5,5,,5,,,
controls,heldout-one-off-exception,1,memory_off,5,0,,0,5,0,,0,default|cc|exception|no_cc|scope,,default|cc|exception|no_cc|scope
controls,heldout-one-off-exception,1,normal,5,4,,5,5,5,,5,,,
controls,heldout-request-not-completion,1,facts_only,6,6,,6,6,6,,6,,,
controls,heldout-request-not-completion,1,memory_off,6,2,,2,6,2,,2,amount|currency|unpaid|unapproved,,amount|currency|unpaid|unapproved
controls,heldout-request-not-completion,1,normal,6,6,,6,6,6,,6,,,
controls,heldout-subject-boundaries,1,facts_only,10,10,,10,10,10,,10,,,
controls,heldout-subject-boundaries,1,memory_off,10,2,,2,10,2,,2,chen|elm|euro|french|shah|pine|pound|english,,chen|elm|euro|french|shah|pine|pound|english
controls,heldout-subject-boundaries,1,normal,10,10,,10,10,10,,10,,,
controls,heldout-update-preserve-rest,2,facts_only,4,3,,4,4,3,,4,term,,
controls,heldout-update-preserve-rest,2,memory_off,4,1,,1,4,1,,1,contract|invoice|term,,contract|invoice|term
controls,heldout-update-preserve-rest,2,normal,4,4,,4,4,4,,4,,,
controls,validation-json-other-subjects,1,facts_only,5,4,,4,5,5,,5,,,
controls,validation-json-other-subjects,1,memory_off,5,1,,1,5,1,,1,cedar_term|cedar_currency|pine_term|pine_currency,,cedar_term|cedar_currency|pine_term|pine_currency
controls,validation-json-other-subjects,1,normal,5,4,,4,5,5,,5,,,
controls,validation-json-shop-policy,1,facts_only,4,4,,4,4,4,,4,,,
controls,validation-json-shop-policy,1,memory_off,4,0,,0,4,0,,0,unit|interval|initials|date,,unit|interval|initials|date
controls,validation-json-shop-policy,1,normal,4,4,,4,4,4,,4,,,
controls,validation-nested-exception,1,facts_only,5,5,,5,5,5,,5,,,
controls,validation-nested-exception,1,memory_off,5,0,,0,5,0,,0,pilot|regular|no_tracking|rounds|not_sent,,pilot|regular|no_tracking|rounds|not_sent
controls,validation-nested-exception,1,normal,5,5,,5,5,5,,5,,,
controls,validation-text-supplier,1,facts_only,11,9,,10,11,9,,11,pending|units,,
controls,validation-text-supplier,1,memory_off,11,0,,0,11,0,,0,owner|copper|supplier|quantity|price|currency|pending|approval|nylon|interval|units,,owner|copper|supplier|quantity|price|currency|pending|approval|nylon|interval|units
controls,validation-text-supplier,1,normal,11,10,,10,11,11,,11,,,