Results with select cases

You may not want to create a result containing the runs for every test case. Instead, you may want to create a result with only a subset of test cases.

For example, if your pipeline degrades on Japanese language queries, you may want to create a result with those test cases only.

TypeScript

Python

submitTestResult() / submit_test_result()

View documentation for simple submission of test results here. You can use this function in a couple ways to accomplish the same goal.

Filter TestCase list

typescript

typescript
import { getTestCases, init, submitTestResult } from "@gentrace/core";
import { aiPipeline } from "../pipelines"; // TODO: replace with your own pipeline
 
init({
  apiKey: process.env.GENTRACE_API_KEY ?? "",
});
 
const PIPELINE_SLUG = "guess-the-year";
 
const japaneseCases = (await getTestCases(PIPELINE_SLUG))
  .filter(tc => tc.name.startsWith("Japanese"));
 
const outputsList : Record<String, any> = []
 
for (const testCase of japaneseCases) {
  const outputs = aiPipeline(testCase);
  outputsList.push(outputs);
}
 
await submitTestResult(
  PIPELINE_SLUG,
  japaneseCases,
  outputs
);

python

python
import os
import gentrace
from app.pipelines import create_ai_output # TODO: replace with your own pipeline
gentrace.init(
    api_key=os.getenv("GENTRACE_API_KEY"),
)
PIPELINE_SLUG = "example-pipeline"
cases = [case for case in gentrace.get_test_cases(pipeline_slug=PIPELINE_SLUG) if case.get("name").startswith("Japanese")]
outputs_list = []
for case in cases:
    outputs_list.append({
        "value": create_ai_output(case["inputs"])
    })
# Creates result only for Japanese cases
result = gentrace.submit_test_result(
    PIPELINE_SLUG, test_cases=cases, outputs_list=outputs_list
)
print("Test result ID:", result["resultId"])

Pull specific TestCase

typescript

typescript
import { getTestCase, init, submitTestResult } from "@gentrace/core";
import { aiPipeline } from "../pipelines"; // TODO: replace with your own pipeline
 
init({
  apiKey: process.env.GENTRACE_API_KEY
});
 
const PIPELINE_SLUG = "guess-the-year";
 
const TEST_CASE_ID = "38485a80-4291-4f33-8797-36d4e9f9ad3f";
 
const failingCase = await getTestCase(TEST_CASE_ID);
 
const caseOutputs : Record<string, any> = aiPipeline(failingCase);
 
// Creates result only for this failing case
await submitTestResult(
  PIPELINE_SLUG,
  [failingCase],
  [caseOutputs],
);

python

python
import os
import gentrace
from app.pipelines import create_ai_output # TODO: replace with your own pipeline
gentrace.init(api_key=os.getenv("GENTRACE_API_KEY"))
PIPELINE_SLUG = "example-pipeline"
TEST_CASE_ID = "38485a80-4291-4f33-8797-36d4e9f9ad3f"
failing_case = gentrace.get_test_case(TEST_CASE_ID)
case_outputs = create_ai_output(failing_case.get("inputs"))
# Creates result only for this failing case
result = gentrace.submit_test_result(
    PIPELINE_SLUG, test_cases=[failing_case], outputs_list=[case_outputs]
)
print("Test result ID:", result["resultId"])

runTest() / run_test()

View documentation for the callback submission of test results here. You can use a filter function to specify the cases you want to run.

typescript

typescript
await runTest(
  PIPELINE_SLUG,
  async (testCase) => {
    const runner = pipeline.start();
 
    const outputs = await runner.measure(
      (inputs) => {
        return {
          yourOutputKey: "Your output value",
        }
      },
      [testCase.inputs],
    );
 
    await runner.submit();
 
    return [outputs, runner];
  },
  (testCase) => testCase.name.startsWith("Japanese test case:")
);

python

python
import os
import gentrace
PIPELINE_SLUG = "guess-the-year"
gentrace.init(
    api_key=os.getenv("GENTRACE_API_KEY"),
)
pipeline = gentrace.Pipeline(
    PIPELINE_SLUG,
    openai_config={
        "api_key": os.getenv("OPENAI_KEY"),
    },
)
def create_embedding_callback(test_case):
    runner = pipeline.start()
    openai_handle = runner.get_openai()
    output = openai_handle.embeddings.create(
        input="sample text", model="text-similarity-davinci-001"
    )
    return [output, runner]
result = gentrace.run_test(
    PIPELINE_SLUG,
    create_embedding_callback,
    # TODO: define your filter function here
    case_filter=lambda x: x.get("name").startswith("Japanese test case:")
)
print("Result: ", result)

The specified callback will only fire for test cases that match the filter function.

If you want to run the callback only on a single test case, you can write a modified filter function that only returns true for that test case.

typescript

typescript
await runTest(
  PIPELINE_SLUG,
  async (testCase) => {
    const runner = pipeline.start();
 
    const outputs = await runner.measure(
      (inputs) => {
        return {
          yourOutputKey: "Your output value",
        }
      },
      [testCase.inputs],
    );
 
    await runner.submit();
 
    return [outputs, runner];
  },
  (testCase) => testCase.id === "38485a80-4291-4f33-8797-36d4e9f9ad3f"
      
(parameter) testCase: Omit<TestCase, "createdAt" | "updatedAt" | "archivedAt">
);

python

python
result = gentrace.run_test(
    PIPELINE_SLUG,
    create_embedding_callback,
    case_filter=lambda x: x.get("id") == "38485a80-4291-4f33-8797-36d4e9f9ad3f"
)
print("Result: ", result)