Flaky test hunter

/** Find flaky tests in the current Git repository. */
import { execFileSync } from 'node:child_process';
import { mkdir, writeFile } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import OpenAI from 'openai';

const client = new OpenAI({ apiKey: process.env.ORCA_API_KEY!, baseURL: process.env.ORCA_BASE_URL! });
const agents = client.beta.agents;
const sessions = agents.sessions;

// How many times to run the suite. Start with RUNS=1 to check that it installs and runs.
const RUNS = Number(process.env.RUNS ?? '5');
// Orca refuses to stage any one file over 16 MiB, however it was uploaded.
const STAGED_FILE_LIMIT = 16 * 1024 * 1024;
const INSTRUCTIONS = `You find flaky tests. You work in a Linux sandbox with a shell.
The repository is in /workspace/repo.tar.gz. Extract it to /workspace/repo first.
Create /workspace/outputs if it does not exist.
Install its test dependencies, then run the test suite the number of times the user asks,
recording which tests fail on each run. Save the raw results to /workspace/outputs/runs/.
A flaky test fails on some runs and passes on others. A test that fails on every run is
broken, not flaky; list it separately and do not try to fix it.
For each flaky test, read the test and the code it covers and name the likely cause:
timing, ordering, shared state, randomness, time zones, or network.
Write a minimal fix as a unified diff to /workspace/outputs/patches/<test-name>.diff,
then rerun that test with the fix applied to check it.
Write /workspace/outputs/flaky_report.md: a table of test, failure rate, cause,
confidence, and whether the fix held. Never claim a fix works unless you reran it.
Do not use emojis.`;

// Committed files only: no .env files, build output, or local secrets.
const snapshot = execFileSync('git', ['archive', '--format=tar.gz', 'HEAD'], { maxBuffer: 512 * 1024 * 1024 });
if (snapshot.length > STAGED_FILE_LIMIT) {
  console.error('Snapshot is over the 16 MiB limit for one staged file. Archive a subfolder instead.');
  process.exit(1);
}

let uploaded: { id: string } | undefined;
let agent: { id: string } | undefined;
let session: { id: string } | undefined;
try {
  uploaded = await client.files.create({ file: new File([snapshot], 'repo.tar.gz'), purpose: 'user_data' });
  agent = await agents.create({ model: process.env.ORCA_MODEL!, name: 'flaky-hunter', instructions: INSTRUCTIONS });
  session = await sessions.create({
    agent_id: agent.id,
    environment: {
      type: 'openai_hosted',
      // Enabled so the agent can install dependencies. Use "disabled" if your
      // tests need nothing from the internet and the image already has them.
      network: { access: 'enabled' },
      files: [{ type: 'file_id', file_id: uploaded.id, path: '/workspace/repo.tar.gz' }],
    },
  });

  const input = `Run the full test suite ${RUNS} times and find the flaky tests.`;
  for await (const event of sessions.stream(session.id, { input })) {
    if (event.type === 'agent.session.turn.completed') console.log('turn:', event.turn.status);
  }

  for await (const artifact of sessions.artifacts.list(session.id)) {
    const content = await sessions.artifacts.content(artifact.id, { session_id: session.id });
    const target = join('flaky-output', artifact.path.split('/outputs/').pop()!);
    await mkdir(dirname(target), { recursive: true });
    await writeFile(target, Buffer.from(await content.arrayBuffer()));
    console.log('saved:', target);
  }
} finally {
  // Delete each resource on its own, so one failed delete does not skip the rest.
  if (session) await sessions.delete(session.id).catch(() => {});
  if (agent) await agents.delete(agent.id).catch(() => {});
  if (uploaded) await client.files.delete(uploaded.id).catch(() => {});
}