Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

chore(test-suite): remove unnecessary stuff

mogery20b2f5f79b86
diff --git a/apps/test-suite/package.json b/apps/test-suite/package.json
index 1419dfc26..7e0ba6d34 100644
--- a/apps/test-suite/package.json
+++ b/apps/test-suite/package.json
@@ -3,33 +3,13 @@
"version": "1.0.0",
"description": "",
"scripts": {
- "test:suite": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false",
- "test:load": "artillery run --output ./load-test-results/test-run-report.json load-test.yml",
- "test:scrape": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathPattern=tests/scrape.test.ts",
- "test:crawl": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathPattern=tests/crawl.test.ts",
- "test:schema-validation": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathPattern=tests/schema-validation.test.ts"
+ "test:load": "artillery run --output ./load-test-results/test-run-report.json load-test.yml"
},
"author": "",
"license": "ISC",
- "dependencies": {
- "@anthropic-ai/sdk": "^0.57.0",
- "@dqbd/tiktoken": "^1.0.14",
- "@supabase/supabase-js": "^2.43.1",
- "axios": "^1.12.2",
- "dotenv": "^16.4.5",
- "jest": "^30.0.5",
- "openai": "^5.10.2",
- "playwright": "^1.56.1",
- "supertest": "^7.1.4",
- "ts-jest": "^29.4.0"
- },
+ "dependencies": {},
"devDependencies": {
- "@jest/globals": "^30.0.5",
- "@types/jest": "^30.0.0",
- "@types/supertest": "^6.0.2",
- "artillery": "^2.0.23",
- "typescript": "^5.4.5",
- "zod": "^3.24.1"
+ "artillery": "^2.0.23"
},
"pnpm": {
"overrides": {
diff --git a/apps/test-suite/tests/crawl.test.ts b/apps/test-suite/tests/crawl.test.ts
deleted file mode 100644
index 533bade39..000000000
--- a/apps/test-suite/tests/crawl.test.ts
+++ /dev/null
@@ -1,150 +0,0 @@
-import request from "supertest";
-import dotenv from "dotenv";
-import { WebsiteScrapeError } from "../utils/types";
-import { logErrors } from "../utils/log";
-
-import websitesData from "../data/crawl.json";
-import "dotenv/config";
-
-import fs from 'fs';
-dotenv.config();
-
-interface WebsiteData {
- website: string;
- expected_min_num_of_pages: number;
- expected_crawled_pages: string[];
-}
-
-const TEST_URL = "http://127.0.0.1:3002";
-
-describe("Crawling Checkup (E2E)", () => {
- beforeAll(() => {
- if (!process.env.TEST_API_KEY) {
- throw new Error("TEST_API_KEY is not set");
- }
- });
-
- describe("Crawling website tests with a dataset", () => {
- it("Should crawl the website and verify the response", async () => {
- let passedTests = 0;
- const startTime = new Date().getTime();
- const date = new Date();
- const logsDir = `logs/${date.getMonth() + 1}-${date.getDate()}-${date.getFullYear()}`;
-
- let errorLogFileName = `${logsDir}/run.log_${new Date().toTimeString().split(' ')[0]}`;
- const errorLog: WebsiteScrapeError[] = [];
-
- for (const websiteData of websitesData) {
- try {
- const crawlResponse = await request(TEST_URL || "")
- .post("/v0/crawl")
- .set("Content-Type", "application/json")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
- .send({ url: websiteData.website, pageOptions: { onlyMainContent: true }, crawlerOptions: { limit: 100, returnOnlyUrls: true }});
-
- const jobId = crawlResponse.body.jobId;
- let completedResponse: any;
- let isFinished = false;
-
- while (!isFinished) {
- completedResponse = await request(TEST_URL)
- .get(`/v0/crawl/status/${jobId}`)
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`);
-
- isFinished = completedResponse.body.status === "completed";
-
- if (!isFinished) {
- await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for 1 second before checking again
- }
- }
-
- if(!completedResponse) {
- // fail the test
- console.log('No response');
- continue;
- // continue;
- }
-
- if (!completedResponse.body || completedResponse.body.status !== "completed") {
- errorLog.push({
- website: websiteData.website,
- prompt: 'CRAWL',
- expected_output: 'SUCCESS',
- actual_output: 'FAILURE',
- error: `Crawl job did not complete successfully.`
- });
- continue;
- }
-
- // check how many webpages were crawled successfully
- // compares with expected_num_of_pages
- if (completedResponse.body.data.length < websiteData.expected_min_num_of_pages) {
- errorLog.push({
- website: websiteData.website,
- prompt: 'CRAWL',
- expected_output: `SUCCESS: ${websiteData.expected_min_num_of_pages}`,
- actual_output: `FAILURE: ${completedResponse.body.data.length}`,
- error: `Expected at least ${websiteData.expected_min_num_of_pages} webpages, but got ${completedResponse.body.data.length}`
- });
- console.log('Error: ', errorLog);
- continue;
- }
-
- // checks if crawled pages contain expected_crawled_pages
- if (websiteData.expected_crawled_pages && websiteData.expected_crawled_pages.length > 0 && websiteData.expected_crawled_pages.some(page => !completedResponse.body.data?.some((d: { url: string }) => d.url === page))) {
- errorLog.push({
- website: websiteData.website,
- prompt: 'CRAWL',
- expected_output: `SUCCESS: ${websiteData.expected_crawled_pages}`,
- actual_output: `FAILURE: ${completedResponse.body.data.map((d: { url: string }) => d.url)}`,
- error: `Expected crawled pages to contain ${websiteData.expected_crawled_pages}, but got ${completedResponse.body.data.map((d: { url: string }) => d.url)}`
- });
- console.log('Error: ', errorLog);
- continue;
- }
-
- // checks if crawled pages not contain expected_not_crawled_pages
- if (websiteData.expected_not_crawled_pages && websiteData.expected_not_crawled_pages.length > 0 && completedResponse.body.data && websiteData.expected_not_crawled_pages.filter(page => completedResponse.body.data.some((d: { url: string }) => d.url === page)).length > 0) {
- errorLog.push({
- website: websiteData.website,
- prompt: 'CRAWL',
- expected_output: `SUCCESS: ${websiteData.expected_not_crawled_pages}`,
- actual_output: `FAILURE: ${completedResponse.body.data.map((d: { url: string }) => d.url)}`,
- error: `Expected crawled pages to not contain ${websiteData.expected_not_crawled_pages}, but got ${completedResponse.body.data.map((d: { url: string }) => d.url)}`
- });
- console.log('Error: ', errorLog);
- continue;
- }
-
- passedTests++;
- } catch (error) {
- console.error(`Error processing ${websiteData.website}: ${error}`);
- errorLog.push({
- website: websiteData.website,
- prompt: 'CRAWL',
- expected_output: 'SUCCESS',
- actual_output: 'FAILURE',
- error: `Error processing ${websiteData.website}: ${error}`
- });
- continue;
- }
- }
-
- const score = (passedTests / websitesData.length) * 100;
- const endTime = new Date().getTime();
- const timeTaken = (endTime - startTime) / 1000;
- console.log(`Score: ${score}%`);
-
- await logErrors(errorLog, timeTaken, 0, score, websitesData.length);
-
- if (process.env.ENV === "local" && errorLog.length > 0) {
- if (!fs.existsSync(logsDir)){
- fs.mkdirSync(logsDir, { recursive: true });
- }
- fs.writeFileSync(errorLogFileName, JSON.stringify(errorLog, null, 2));
- }
-
- expect(score).toBeGreaterThanOrEqual(90);
- }, 350000); // 150 seconds timeout
- });
-});
diff --git a/apps/test-suite/tests/schema-validation.test.ts b/apps/test-suite/tests/schema-validation.test.ts
deleted file mode 100644
index c0418c0ee..000000000
--- a/apps/test-suite/tests/schema-validation.test.ts
+++ /dev/null
@@ -1,271 +0,0 @@
-import {actionsSchema, scrapeOptions} from '../../api/src/controllers/v1/types';
-import {describe, it, expect} from '@jest/globals';
-
-describe('Schema Validation Tests', () => {
- describe('Actions Schema Validation', () => {
- it('should allow valid actions within limits', () => {
- const validActions = [
- {type: 'wait', milliseconds: 1000},
- {type: 'click', selector: '#button'},
- {type: 'screenshot', fullPage: false},
- {type: 'write', text: 'olá - hello'},
- {type: 'press', key: 'Enter'},
- {type: 'scroll', direction: 'down'},
- {type: 'scrape'},
- {type: 'executeJavascript', script: 'console.log("test")'},
- ];
-
- const result = actionsSchema.safeParse(validActions);
- expect(result.success).toBe(true);
- });
-
- it('should reject more than 15 actions', () => {
- const tooManyActions = Array(16).fill({type: 'click', selector: '#button'});
-
- const result = actionsSchema.safeParse(tooManyActions);
- expect(result.success).toBe(false);
- if (!result.success) {
- expect(result.error.errors[0].message).toBe('Maximum of 15 actions allowed');
- }
- });
-
- describe('Wait Action Validations', () => {
- it('should validate wait with milliseconds', () => {
- const validWait = [{type: 'wait', milliseconds: 1000}];
- expect(actionsSchema.safeParse(validWait).success).toBe(true);
-
- const invalidWait = [{type: 'wait', milliseconds: -1000}];
- expect(actionsSchema.safeParse(invalidWait).success).toBe(false);
- });
-
- it('should validate wait with selector', () => {
- const validWait = [{type: 'wait', selector: '#element'}];
- expect(actionsSchema.safeParse(validWait).success).toBe(true);
- });
-
- it('should reject wait with both milliseconds and selector', () => {
- const invalidWait = [{type: 'wait', milliseconds: 1000, selector: '#element'}];
- const result = actionsSchema.safeParse(invalidWait);
- expect(result.success).toBe(false);
- if (!result.success) {
- expect(result.error.errors[0].message).toBe(
- "Either 'milliseconds' or 'selector' must be provided, but not both.",
- );
- }
- });
-
- it('should reject wait without either milliseconds or selector', () => {
- const invalidWait = [{type: 'wait'}];
- const result = actionsSchema.safeParse(invalidWait);
- expect(result.success).toBe(false);
- });
-
- it('should reject when total wait time exceeds 60 seconds', () => {
- const longWaitActions = [
- {type: 'wait', milliseconds: 50000},
- {type: 'wait', milliseconds: 11000},
- ];
-
- const result = actionsSchema.safeParse(longWaitActions);
- expect(result.success).toBe(false);
- if (!result.success) {
- expect(result.error.errors[0].message).toBe(
- 'Total wait time (waitFor + wait actions) cannot exceed 60 seconds',
- );
- }
- });
-
- it('should count selector waits as 1 second each', () => {
- // 15 selector clicks = 15 seconds total, should pass
- const maxWaitSelectors = Array(15).fill({type: 'click', selector: '#element'});
- const result = actionsSchema.safeParse(maxWaitSelectors);
- expect(result.success).toBe(true);
-
- // Test the time limit with mixed waits
- const mixedWaits = [
- {type: 'wait', milliseconds: 58000}, // 58 seconds
- {type: 'wait', selector: '#element'}, // 1 second
- {type: 'wait', selector: '#load-more-button'}, // 1 second
- {type: 'wait', selector: '#toomuch'}, // 1 second, exceeds 60 seconds total
- ];
- const timeFailResult = actionsSchema.safeParse(mixedWaits);
- expect(timeFailResult.success).toBe(false);
- if (!timeFailResult.success) {
- expect(timeFailResult.error.errors[0].message).toBe(
- 'Total wait time (waitFor + wait actions) cannot exceed 60 seconds',
- );
- }
- });
- });
-
- describe('Other Action Type Validations', () => {
- it('should validate click action', () => {
- const validClick = [{type: 'click', selector: '#button'}];
- expect(actionsSchema.safeParse(validClick).success).toBe(true);
-
- const invalidClick = [{type: 'click'}];
- expect(actionsSchema.safeParse(invalidClick).success).toBe(false);
- });
-
- it('should validate screenshot action', () => {
- const validScreenshot = [{type: 'screenshot', fullPage: true}];
- expect(actionsSchema.safeParse(validScreenshot).success).toBe(true);
-
- const defaultScreenshot = [{type: 'screenshot', fullPage: false}];
- const result = actionsSchema.safeParse(defaultScreenshot);
- expect(result.success).toBe(true);
- });
-
- it('should validate write action', () => {
- const validWrite = [{type: 'write', text: 'hello'}];
- expect(actionsSchema.safeParse(validWrite).success).toBe(true);
-
- const invalidWrite = [{type: 'write'}];
- expect(actionsSchema.safeParse(invalidWrite).success).toBe(false);
- });
-
- it('should validate press action', () => {
- const validPress = [{type: 'press', key: 'Enter'}];
- expect(actionsSchema.safeParse(validPress).success).toBe(true);
-
- const invalidPress = [{type: 'press'}];
- expect(actionsSchema.safeParse(invalidPress).success).toBe(false);
- });
-
- it('should validate scroll action', () => {
- const validScroll = [{type: 'scroll', direction: 'up', selector: '#element'}];
- expect(actionsSchema.safeParse(validScroll).success).toBe(true);
-
- const defaultScroll = [{type: 'scroll', direction: 'down'}];
- const result = actionsSchema.safeParse(defaultScroll);
- expect(result.success).toBe(true);
-
- const invalidDirection = [{type: 'scroll', direction: 'left'}];
- expect(actionsSchema.safeParse(invalidDirection).success).toBe(false);
- });
-
- it('should validate scrape action', () => {
- const validScrape = [{type: 'scrape'}];
- expect(actionsSchema.safeParse(validScrape).success).toBe(true);
- });
-
- it('should validate executeJavascript action', () => {
- const validJs = [{type: 'executeJavascript', script: 'console.log("test")'}];
- expect(actionsSchema.safeParse(validJs).success).toBe(true);
-
- const invalidJs = [{type: 'executeJavascript'}];
- expect(actionsSchema.safeParse(invalidJs).success).toBe(false);
- });
- });
- });
-
- describe('Scrape Options Schema Validation', () => {
- it('should validate waitFor limit', () => {
- const validOptions = {
- waitFor: 60000, // 60 seconds
- };
- expect(scrapeOptions.safeParse(validOptions).success).toBe(true);
-
- const invalidOptions = {
- waitFor: 61000, // 61 seconds
- };
- expect(scrapeOptions.safeParse(invalidOptions).success).toBe(false);
- });
-
- describe('Combined Wait Time Validations', () => {
- it('should validate combined waitFor and actions wait time', () => {
- // Test valid combination (at the limit)
- const validOptions = {
- waitFor: 30000, // 30 seconds
- actions: [
- {type: 'wait', milliseconds: 29000}, // 29 seconds
- {type: 'wait', selector: '#element'}, // 1 second
- ],
- };
- expect(scrapeOptions.safeParse(validOptions).success).toBe(true);
-
- // Test invalid combination (exceeds limit)
- const invalidOptions = {
- waitFor: 30000, // 30 seconds
- actions: [
- {type: 'wait', milliseconds: 29000}, // 29 seconds
- {type: 'wait', selector: '#element'}, // 1 second
- {type: 'wait', selector: '#another'}, // 1 second
- ],
- };
- const failResult = scrapeOptions.safeParse(invalidOptions);
- expect(failResult.success).toBe(false);
- if (!failResult.success) {
- expect(failResult.error.errors[0].message).toBe(
- 'Total wait time (waitFor + wait actions) cannot exceed 60 seconds',
- );
- }
- });
-
- it('should handle edge cases of combined wait times', () => {
- // Test with only waitFor at limit
- const maxWaitFor = {
- waitFor: 60000, // 60 seconds
- actions: [
- {type: 'write', text: 'Olá galera!'}, // non-wait action
- ],
- };
- expect(scrapeOptions.safeParse(maxWaitFor).success).toBe(true);
-
- // Test with only action waits at limit
- const maxActionWaits = {
- waitFor: 0,
- actions: [
- {type: 'wait', milliseconds: 59000}, // 59 seconds
- {type: 'wait', selector: '#element'}, // 1 second
- ],
- };
- expect(scrapeOptions.safeParse(maxActionWaits).success).toBe(true);
-
- // Test with mixed waits slightly over limit
- const slightlyOver = {
- waitFor: 30000, // 30 seconds
- actions: [
- {type: 'wait', milliseconds: 30000}, // 30 seconds
- {type: 'wait', selector: '#element'}, // 1 second
- ],
- };
- const overResult = scrapeOptions.safeParse(slightlyOver);
- expect(overResult.success).toBe(false);
- if (!overResult.success) {
- expect(overResult.error.errors[0].message).toBe(
- 'Total wait time (waitFor + wait actions) cannot exceed 60 seconds',
- );
- }
- });
- });
-
- describe('Format Validations', () => {
- it('should validate screenshot format combinations', () => {
- const validScreenshot = {
- formats: ['screenshot'],
- };
- expect(scrapeOptions.safeParse(validScreenshot).success).toBe(true);
-
- const validFullPage = {
- formats: ['screenshot@fullPage'],
- };
- expect(scrapeOptions.safeParse(validFullPage).success).toBe(true);
-
- const invalidBoth = {
- formats: ['screenshot', 'screenshot@fullPage'],
- };
- expect(scrapeOptions.safeParse(invalidBoth).success).toBe(false);
- });
-
- it('should default to markdown format', () => {
- const noFormat = {};
- const result = scrapeOptions.safeParse(noFormat);
- expect(result.success).toBe(true);
- if (result.success) {
- expect(result.data.formats).toEqual(['markdown']);
- }
- });
- });
- });
-});
diff --git a/apps/test-suite/tests/scrape.test.ts b/apps/test-suite/tests/scrape.test.ts
deleted file mode 100644
index 776c59fe5..000000000
--- a/apps/test-suite/tests/scrape.test.ts
+++ /dev/null
@@ -1,189 +0,0 @@
-import request from "supertest";
-import dotenv from "dotenv";
-import { numTokensFromString } from "../utils/tokens";
-import OpenAI from "openai";
-import { WebsiteScrapeError } from "../utils/types";
-import { logErrors } from "../utils/log";
-
-import websitesData from "../data/scrape.json";
-import "dotenv/config";
-
-import fs from 'fs';
-dotenv.config();
-
-interface WebsiteData {
- website: string;
- prompt: string;
- expected_output: string;
-}
-
-const TEST_URL = "http://127.0.0.1:3002";
-
-describe("Scraping Checkup (E2E)", () => {
- beforeAll(() => {
- if (!process.env.TEST_API_KEY) {
- throw new Error("TEST_API_KEY is not set");
- }
- if (!process.env.OPENAI_API_KEY) {
- throw new Error("OPENAI_API_KEY is not set");
- }
- });
-
- describe("Scraping website tests with a dataset", () => {
- it("Should scrape the website and prompt it against OpenAI", async () => {
- let totalTimeTaken = 0;
- let passedTests = 0;
- const batchSize = 15; // Adjusted to comply with the rate limit of 15 per minute
- const batchPromises = [];
- let totalTokens = 0;
-
- const startTime = new Date().getTime();
- const date = new Date();
- const logsDir = `logs/${date.getMonth() + 1}-${date.getDate()}-${date.getFullYear()}`;
-
- let errorLogFileName = `${logsDir}/run.log_${new Date().toTimeString().split(' ')[0]}`;
- const errorLog: WebsiteScrapeError[] = [];
-
- for (let i = 0; i < websitesData.length; i += batchSize) {
- // Introducing delay to respect the rate limit of 15 requests per minute
- await new Promise(resolve => setTimeout(resolve, 10000));
-
- const batch = websitesData.slice(i, i + batchSize);
- const batchPromise = Promise.all(
- batch.map(async (websiteData: WebsiteData) => {
- try {
- const startTime = new Date().getTime();
- const scrapedContent = await request(TEST_URL || "")
- .post("/v1/scrape")
- .set("Content-Type", "application/json")
- .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
- .send({ url: websiteData.website });
-
- const endTime = new Date().getTime();
- const timeTaken = endTime - startTime;
- totalTimeTaken += timeTaken;
-
- if (scrapedContent.statusCode !== 200) {
- console.error(`Failed to scrape ${websiteData.website} ${scrapedContent.statusCode}`);
- errorLog.push({
- website: websiteData.website,
- prompt: websiteData.prompt,
- expected_output: websiteData.expected_output,
- actual_output: "",
- error: `Failed to scrape website. ${scrapedContent.statusCode} ${scrapedContent.body.error}`
- });
- return null;
- }
-
- const openai = new OpenAI({
- apiKey: process.env.OPENAI_API_KEY,
- });
-
- const prompt = `Based on this markdown extracted from a website html page, ${websiteData.prompt} Just say 'yes' or 'no' to the question.\nWebsite markdown: ${scrapedContent.body.data.markdown}\n`;
-
- let msg = null;
- const maxRetries = 3;
- let attempts = 0;
- while (!msg && attempts < maxRetries) {
- try {
- msg = await openai.chat.completions.create({
- model: "gpt-4-turbo",
- max_tokens: 100,
- temperature: 0,
- messages: [
- {
- role: "user",
- content: prompt
- },
- ],
- });
- } catch (error) {
- console.error(`Attempt ${attempts + 1}: Failed to prompt for ${websiteData.website}, error: ${error}`);
- attempts++;
- if (attempts < maxRetries) {
- console.log(`Retrying... Attempt ${attempts + 1}`);
- await new Promise(resolve => setTimeout(resolve, 2000)); // Wait for 2 seconds before retrying
- }
- }
- }
-
- if (!msg) {
- console.error(`Failed to prompt for ${websiteData.website} after ${maxRetries} attempts`);
- errorLog.push({
- website: websiteData.website,
- prompt: websiteData.prompt,
- expected_output: websiteData.expected_output,
- actual_output: "",
- error: "Failed to prompt... model error."
- });
- return null;
- }
-
- const actualOutput = (msg.choices[0].message.content ?? "").toLowerCase()
- const expectedOutput = websiteData.expected_output.toLowerCase();
-
- const numTokens = numTokensFromString(prompt, "gpt-4") + numTokensFromString(actualOutput, "gpt-4");
-
- totalTokens += numTokens;
- if (actualOutput.includes(expectedOutput)) {
- passedTests++;
- } else {
- console.error(
- `This website failed the test: ${websiteData.website}`
- );
- console.error(`Actual output: ${actualOutput}`);
- errorLog.push({
- website: websiteData.website,
- prompt: websiteData.prompt,
- expected_output: websiteData.expected_output,
- actual_output: actualOutput,
- error: "Output mismatch"
- });
- }
-
- return {
- website: websiteData.website,
- prompt: websiteData.prompt,
- expectedOutput,
- actualOutput,
- };
- } catch (error) {
- console.error(
- `Error processing ${websiteData.website}: ${error}`
- );
- errorLog.push({
- website: websiteData.website,
- prompt: websiteData.prompt,
- expected_output: websiteData.expected_output,
- actual_output: "",
- error: `Error processing ${websiteData.website}: ${error}`
- });
- return null;
- }
- })
- );
- batchPromises.push(batchPromise);
- }
-
- (await Promise.all(batchPromises)).flat();
- const score = (passedTests / websitesData.length) * 100;
- const endTime = new Date().getTime();
- const timeTaken = (endTime - startTime) / 1000;
- console.log(`Score: ${score}%`);
- console.log(`Total tokens: ${totalTokens}`);
- console.log(`Total time taken: ${totalTimeTaken} miliseconds`);
-
- await logErrors(errorLog, timeTaken, totalTokens, score, websitesData.length);
-
- if (process.env.ENV === "local" && errorLog.length > 0) {
- if (!fs.existsSync(logsDir)) {
- fs.mkdirSync(logsDir, { recursive: true });
- }
- fs.writeFileSync(errorLogFileName, JSON.stringify(errorLog, null, 2));
- }
-
-
- expect(score).toBeGreaterThanOrEqual(70);
- }, 350000); // 150 seconds timeout
- });
-});
diff --git a/apps/test-suite/tsconfig.json b/apps/test-suite/tsconfig.json
deleted file mode 100644
index afa29e712..000000000
--- a/apps/test-suite/tsconfig.json
+++ /dev/null
@@ -1,109 +0,0 @@
-{
- "compilerOptions": {
- /* Visit https://aka.ms/tsconfig to read more about this file */
-
- /* Projects */
- // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
- // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
- // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */
- // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */
- // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
- // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
-
- /* Language and Environment */
- "target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
- // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
- // "jsx": "preserve", /* Specify what JSX code is generated. */
- // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */
- // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */
- // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
- // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
- // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
- // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
- // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
- // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
- // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */
-
- /* Modules */
- "module": "commonjs", /* Specify what module code is generated. */
- // "rootDir": "./", /* Specify the root folder within your source files. */
- // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */
- // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
- // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
- // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
- // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */
- // "types": [], /* Specify type package names to be included without being referenced in a source file. */
- // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
- // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */
- // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */
- // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */
- // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */
- // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */
- "resolveJsonModule": true, /* Enable importing .json files. */
- // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */
- // "noResolve": true, /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
-
- /* JavaScript Support */
- // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
- // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
- // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
-
- /* Emit */
- // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
- // "declarationMap": true, /* Create sourcemaps for d.ts files. */
- // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
- // "sourceMap": true, /* Create source map files for emitted JavaScript files. */
- // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
- // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
- // "outDir": "./", /* Specify an output folder for all emitted files. */
- // "removeComments": true, /* Disable emitting comments. */
- // "noEmit": true, /* Disable emitting files from a compilation. */
- // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
- // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */
- // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
- // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
- // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
- // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
- // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
- // "newLine": "crlf", /* Set the newline character for emitting files. */
- // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
- // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */
- // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
- // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */
- // "declarationDir": "./", /* Specify the output directory for generated declaration files. */
- // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
-
- /* Interop Constraints */
- // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
- // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */
- // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
- "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
- // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
- "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */
-
- /* Type Checking */
- "strict": true, /* Enable all strict type-checking options. */
- // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */
- // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */
- // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
- // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
- // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */
- // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */
- // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */
- // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */
- // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */
- // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */
- // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */
- // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */
- // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */
- // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */
- // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */
- // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */
- // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
- // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
-
- /* Completeness */
- // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
- "skipLibCheck": true /* Skip type checking all .d.ts files. */
- }
-}
diff --git a/apps/test-suite/utils/log.ts b/apps/test-suite/utils/log.ts
deleted file mode 100644
index b029bf71a..000000000
--- a/apps/test-suite/utils/log.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-import { supabase_service } from "./supabase";
-import { WebsiteScrapeError } from "./types";
-
-export async function logErrors(dataError: WebsiteScrapeError[], time_taken: number, num_tokens:number, score: number, num_pages_tested: number,) {
- try {
- await supabase_service.from("test_suite_logs").insert([{log:dataError, time_taken, num_tokens, score, num_pages_tested, is_error: dataError.length > 0}]);
- } catch (error) {
- console.error(`Error logging to supabase: ${error}`);
- }
-}
diff --git a/apps/test-suite/utils/misc.ts b/apps/test-suite/utils/misc.ts
deleted file mode 100644
index 57e2cfd16..000000000
--- a/apps/test-suite/utils/misc.ts
+++ /dev/null
@@ -1,47 +0,0 @@
-const getRandomLinksFromContent = async (options: {
- content: string;
- excludes: string[];
- limit: number;
- }): Promise<string[]> => {
- const regex = /(?<=\()https:\/\/(.*?)(?=\))/g;
- const links = options.content.match(regex);
- const filteredLinks = links
- ? links.filter(
- (link) => !options.excludes.some((exclude) => link.includes(exclude))
- )
- : [];
- const uniqueLinks = [...new Set(filteredLinks)]; // Ensure all links are unique
- const randomLinks = [];
- while (randomLinks.length < options.limit && uniqueLinks.length > 0) {
- const randomIndex = Math.floor(Math.random() * uniqueLinks.length);
- randomLinks.push(uniqueLinks.splice(randomIndex, 1)[0]);
- }
- return randomLinks;
- };
-
- function fuzzyContains(options: {
- largeText: string;
- queryText: string;
- threshold?: number;
- }): boolean {
- // Normalize texts: lowercasing and removing non-alphanumeric characters
- const normalize = (text: string) =>
- text.toLowerCase().replace(/[^a-z0-9]+/g, " ");
-
- const normalizedLargeText = normalize(options.largeText);
- const normalizedQueryText = normalize(options.queryText);
-
- // Split the query into words
- const queryWords = normalizedQueryText.split(/\s+/);
-
- // Count how many query words are in the large text
- const matchCount = queryWords.reduce((count, word) => {
- return count + (normalizedLargeText.includes(word) ? 1 : 0);
- }, 0);
-
- // Calculate the percentage of words matched
- const matchPercentage = matchCount / queryWords.length;
-
- // Check if the match percentage meets or exceeds the threshold
- return matchPercentage >= (options.threshold || 0.8);
- }
\ No newline at end of file
diff --git a/apps/test-suite/utils/supabase.ts b/apps/test-suite/utils/supabase.ts
deleted file mode 100644
index a1549e245..000000000
--- a/apps/test-suite/utils/supabase.ts
+++ /dev/null
@@ -1,65 +0,0 @@
-import { createClient, SupabaseClient } from "@supabase/supabase-js";
-import { configDotenv } from "dotenv";
-configDotenv();
-
-// SupabaseService class initializes the Supabase client conditionally based on environment variables.
-class SupabaseService {
- private client: SupabaseClient | null = null;
-
- constructor() {
- const supabaseUrl = process.env.SUPABASE_URL;
- const supabaseServiceToken = process.env.SUPABASE_SERVICE_TOKEN;
- // Only initialize the Supabase client if both URL and Service Token are provided.
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true';
- if (!useDbAuthentication) {
- // Warn the user that Authentication is disabled by setting the client to null
- console.warn(
- "Authentication is disabled. Supabase client will not be initialized."
- );
- this.client = null;
- } else if (!supabaseUrl || !supabaseServiceToken) {
- console.error(
- "Supabase environment variables aren't configured correctly. Supabase client will not be initialized. Fix ENV configuration or disable DB authentication with USE_DB_AUTHENTICATION env variable"
- );
- } else {
- this.client = createClient(supabaseUrl, supabaseServiceToken);
- }
- }
-
- // Provides access to the initialized Supabase client, if available.
- getClient(): SupabaseClient | null {
- return this.client;
- }
-}
-
-// Using a Proxy to handle dynamic access to the Supabase client or service methods.
-// This approach ensures that if Supabase is not configured, any attempt to use it will result in a clear error.
-export const supabase_service: SupabaseClient = new Proxy(
- new SupabaseService(),
- {
- get: function (target, prop, receiver) {
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === 'true';
- if (!useDbAuthentication) {
- console.debug(
- "Attempted to access Supabase client when it's not configured."
- );
- }
- const client = target.getClient();
- // If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.
- if (client === null) {
- console.error(
- "Attempted to access Supabase client when it's not configured."
- );
- return () => {
- throw new Error("Supabase client is not configured.");
- };
- }
- // Direct access to SupabaseService properties takes precedence.
- if (prop in target) {
- return Reflect.get(target, prop, receiver);
- }
- // Otherwise, delegate access to the Supabase client.
- return Reflect.get(client, prop, receiver);
- },
- }
-) as unknown as SupabaseClient;
diff --git a/apps/test-suite/utils/tokens.ts b/apps/test-suite/utils/tokens.ts
deleted file mode 100644
index f47a6b3c1..000000000
--- a/apps/test-suite/utils/tokens.ts
+++ /dev/null
@@ -1,16 +0,0 @@
-import { encoding_for_model } from "@dqbd/tiktoken";
-import { TiktokenModel } from "@dqbd/tiktoken";
-
-// This function calculates the number of tokens in a text string using GPT-3.5-turbo model
-export function numTokensFromString(message: string, model: string): number {
- const encoder = encoding_for_model(model as TiktokenModel);
-
- // Encode the message into tokens
- const tokens = encoder.encode(message);
-
- // Free the encoder resources after use
- encoder.free();
-
- // Return the number of tokens
- return tokens.length;
-}
diff --git a/apps/test-suite/utils/types.ts b/apps/test-suite/utils/types.ts
deleted file mode 100644
index dced48d7e..000000000
--- a/apps/test-suite/utils/types.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-export interface WebsiteScrapeError {
- website: string;
- prompt: string;
- expected_output: string;
- actual_output: string;
- error: string;
-}