Test runners - Update result with runners

No Python support

We have not implemented this SDK function in Python yet.

The updateTestResultWithRunners() function allows you to update an existing test result with additional case output data. This function is complementary with getTestRunners() and submitTestRunners().

Use cases

• Chunked uploads for large datasets exceeding API gateway limits (e.g., 10 MB for AWS)
• Incremental addition of test case results for long-running processes
• Targeted updates to specific test cases within a larger result set

Parallelization Example

typescript
import {
  init,
  Pipeline,
  PipelineRunTestCaseTuple,
  getTestRunners,
  submitTestRunners,
  updateTestResultWithRunners,
} from '@gentrace/core';
 
function exampleResponse(inputs: any) {
  return 'This is a generated response from the AI';
}
 
// utility function to enable parallelism
export const enableParallelism = async <T, U>(
  items: T[],
  callbackFn: (t: T) => Promise<U>,
  { parallelThreads = 10 }: { parallelThreads?: number } = {},
) => {
  const results = Array<U>(items.length);
 
  const iterator = items.entries();
  const doAction = async (iterator: IterableIterator<[number, T]>) => {
    for (const [index, item] of iterator) {
      results[index] = await callbackFn(item);
    }
  };
  const workers = Array(parallelThreads).fill(iterator).map(doAction);
  await Promise.allSettled(workers);
  return results;
};
 
async function main() {
  init({
    apiKey: process.env.GENTRACE_API_KEY ?? '',
  });
 
  const PIPELINE_SLUG = 'guess-the-year';
 
  // get the existing pipeline (if already exists)
  const pipelineBySlug = new Pipeline({
    slug: PIPELINE_SLUG,
  });
 
  // example pipeline by ID
  const pipelineById = new Pipeline({
    id: 'c10408c7-abde-5c19-b339-e8b1087c9b64',
  });
 
  const pipeline = pipelineBySlug;
 
  const exampleHandler = async ([
    runner,
    testCase,
  ]: PipelineRunTestCaseTuple) => {
    await runner.measure(
      (inputs: any) => {
        return {
          example: exampleResponse(inputs),
        };
      },
      [testCase.inputs],
    );
  };
 
  const pipelineRunTestCases = await getTestRunners(pipeline);
 
  // Create an initial empty test result
  const initialResult = await submitTestRunners(pipeline, []);
  const resultId = initialResult.id;
 
  // Process test cases in batches of 10, mimicking behavior in a distributed datacenter.
  // This approach allows for efficient processing of large datasets by breaking them into
  // manageable chunks, similar to how distributed systems handle data processing across
  // multiple nodes
  const batchSize = 10;
  for (let i = 0; i < pipelineRunTestCases.length; i += batchSize) {
    const batch = pipelineRunTestCases.slice(i, i + batchSize);
 
    await enableParallelism(batch, exampleHandler, {
      parallelThreads: 5,
    });
 
    // Update the test result with the processed batch
    await updateTestResultWithRunners(resultId, batch);
    console.log(`Updated result with batch ${i / batchSize + 1}`);
  }
 
  console.log(`Finished updating result ${resultId}`);
}
 
main();