🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import {
getFernPythonDeprecations,
patchGeneratedPythonDeprecations,
} from "../../../../scripts/python-sdk/patch-generated-deprecations";
const GENERATED_CLIENT = `# This file was auto-generated by Fern from our API Definition.
import typing
class WidgetClient:
def get_thing(self) -> None:
pass
class AsyncWidgetClient:
async def get_thing(self) -> None:
pass
`;
const GENERATED_RAW_CLIENT = `# This file was auto-generated by Fern from our API Definition.
import typing
from json.decoder import JSONDecodeError
class RawWidgetClient:
def get_thing(self) -> None:
pass
class AsyncRawWidgetClient:
async def get_thing(self) -> None:
pass
`;
function withFixture(
availability: string,
assertions: (definitionDirectory: string, apiRoot: string) => void,
) {
const root = fs.mkdtempSync(
path.join(os.tmpdir(), "python-sdk-deprecations-"),
);
const definitionDirectory = path.join(root, "definition");
const resourceDirectory = path.join(root, "api", "legacy", "widget_client");
try {
fs.mkdirSync(path.join(definitionDirectory, "legacy"), {
recursive: true,
});
fs.mkdirSync(resourceDirectory, { recursive: true });
fs.writeFileSync(path.join(definitionDirectory, "api.yml"), "name: api\n");
fs.writeFileSync(
path.join(definitionDirectory, "legacy", "widget-client.yml"),
[
"service:",
" endpoints:",
" getThing:",
" availability:",
...availability.split("\n").map((line) => ` ${line}`),
" method: GET",
" path: /widgets/{widgetId}",
" request:",
" body:",
" properties:",
" oldField:",
" type: optional<string>",
" availability:",
" status: deprecated",
" message: This field-level marker is not an endpoint.",
"",
].join("\n"),
);
fs.writeFileSync(
path.join(resourceDirectory, "client.py"),
GENERATED_CLIENT,
);
fs.writeFileSync(
path.join(resourceDirectory, "raw_client.py"),
GENERATED_RAW_CLIENT,
);
assertions(definitionDirectory, path.join(root, "api"));
} finally {
fs.rmSync(root, { recursive: true });
}
}
describe("Python SDK deprecations", () => {
it("maps the current Fern endpoint deprecations to Python resource methods", () => {
const definitionDirectory = path.resolve(
process.cwd(),
"../fern/apis/server/definition",
);
expect(
getFernPythonDeprecations(definitionDirectory).map(
({ resourcePath, methodName }) =>
`${resourcePath.replaceAll(path.sep, "/")}:${methodName}`,
),
).toEqual([]);
});
it("decorates sync, async, raw, and async-raw endpoint methods", () => {
const message =
'Use "new_api" at C:\\sdk instead.\nThen call the client\'s replacement.';
withFixture(
[
"status: deprecated",
"message: |-",
' Use "new_api" at C:\\sdk instead.',
" Then call the client's replacement.",
].join("\n"),
(definitionDirectory, apiRoot) => {
expect(getFernPythonDeprecations(definitionDirectory)).toEqual([
{
definitionPath: path.join(
definitionDirectory,
"legacy",
"widget-client.yml",
),
endpointName: "getThing",
resourcePath: path.join("legacy", "widget_client"),
methodName: "get_thing",
message,
},
]);
expect(
patchGeneratedPythonDeprecations({ definitionDirectory, apiRoot }),
).toEqual({ changedFiles: 2, decoratedMethods: 4 });
const decorator = ` @typing_extensions.deprecated(${JSON.stringify(message)}, category=None)`;
for (const fileName of ["client.py", "raw_client.py"]) {
const contents = fs.readFileSync(
path.join(apiRoot, "legacy", "widget_client", fileName),
"utf8",
);
expect(contents.match(/import typing_extensions/g)).toHaveLength(1);
expect(
contents.match(/@typing_extensions\.deprecated\(/g),
).toHaveLength(2);
expect(contents).toContain(`${decorator}\n def get_thing(`);
expect(contents).toContain(`${decorator}\n async def get_thing(`);
}
expect(
patchGeneratedPythonDeprecations({ definitionDirectory, apiRoot }),
).toEqual({ changedFiles: 0, decoratedMethods: 0 });
},
);
});
it("refuses an endpoint deprecation without a customer-facing message", () => {
withFixture("status: deprecated", (definitionDirectory) => {
expect(() => getFernPythonDeprecations(definitionDirectory)).toThrow(
/must define availability\.message/,
);
});
});
it("refuses to patch an incomplete generated client surface", () => {
withFixture(
"status: deprecated\nmessage: Use the replacement endpoint.",
(definitionDirectory, apiRoot) => {
const clientPath = path.join(
apiRoot,
"legacy",
"widget_client",
"client.py",
);
const rawClientPath = path.join(
apiRoot,
"legacy",
"widget_client",
"raw_client.py",
);
const incompleteRawClient = GENERATED_RAW_CLIENT.replace(
" async def get_thing(",
" async def other(",
);
fs.writeFileSync(rawClientPath, incompleteRawClient);
expect(() =>
patchGeneratedPythonDeprecations({ definitionDirectory, apiRoot }),
).toThrow(/expected one sync and one async get_thing method/);
expect(fs.readFileSync(clientPath, "utf8")).toBe(GENERATED_CLIENT);
expect(fs.readFileSync(rawClientPath, "utf8")).toBe(
incompleteRawClient,
);
},
);
});
});