bugbot_json_values.test.ts

import {createHash} from 'node:crypto';
import {readFileSync} from 'node:fs';
import {join} from 'node:path';
import {getBugbotPrompt, type BugbotParams} from '../../prompts/bugbot';
import {buildBugbotBenchmarkPrompt, runBugbotBenchmarkAgent} from '../bugbot_benchmark_runner';
import {evaluateBugbotBenchmark, loadBugbotBenchmark} from '../bugbot_benchmark';
import {buildLiteralVerificationPrompt, verifyGeneratedLiteralFindings} from '../../application/usecases/steps/commit/bugbot/verify_generated_literal_findings';
import {MAX_LITERAL_VERIFICATION_PROMPT_BYTES} from '../../application/policies/bugbot_literal_verification_policy';
import {normalizeBugbotResponse} from '../../application/usecases/steps/commit/bugbot/prepare_bugbot_findings_policy';
import type {FindingsQueryRequest} from '../../application/ports/agent_findings_ports';

const root = join(__dirname, '../../..');
const corpusPath = join(root, 'quality/bugbot/json-value-corpus.v1.json');
const read = (path: string) => readFileSync(join(root, path), 'utf8');
const hash = (text: string) => createHash('sha256').update(text).digest('hex');
const evidence = JSON.parse(read('quality/bugbot/evidence/issue-855.json')) as {
  ownedProbe: {path: string; probeSha256: string; runnerPath: string; runnerSha256: string; python: string;
    castIdentityObserved: boolean; observations: Array<{id: string; startLine: number; endLine: number;
      valueType?: string; value?: unknown; exception?: string}>};
  qualification: {modelInputSha256: string}; preservedInputs: Record<string, string>;
};
const params: BugbotParams = {
  owner: 'owned', repo: 'fixture', headBranch: 'bugfix/855', baseBranch: 'develop', issueNumber: '855',
  projectContextInstruction: '', changeScopeInstruction: 'Review supplied source.', targetLocale: 'en-US',
  ignoreBlock: '', previousBlock: '', coverageBlock: '', diffBlock: '+return read_object(json.dumps(row["entry"]))',
};

it.each([
  ['decoded_object', false], ['nested_outer_object', false], ['text_decoder', false],
  ['guarded_text_cast', false], ['unicode_nested_value', false], ['separate_serialization_levels', false],
  ['object_cast_into_text_reader', true], ['text_encoded_twice', true], ['raw_object_into_text_reader', true],
  ['python_repr_as_json', true], ['missing_inner_decode', true], ['annotation_is_not_conversion', true],
] as const)('JSON boundary %s binds exact source, actual values and opposite-verdict scoring', async (name, actionable) => {
  const source = read(evidence.ownedProbe.path);
  expect(hash(source)).toBe(evidence.ownedProbe.probeSha256);
  const observation = evidence.ownedProbe.observations.find(value => value.id === name)!;
  const corpus = await loadBugbotBenchmark(corpusPath);
  const testCase = corpus.cases.find(value => value.id === 'json-values-' + name)!;
  const selected = source.split('\n').slice(observation.startLine - 1, observation.endLine).join('\n');
  expect(selected.startsWith('def ' + name + '():')).toBe(true);
  expect(testCase.startLine).toBe(observation.startLine);
  expect(testCase.diff.split('\n').map(line => line.slice(1)).join('\n')).toBe(selected);
  expect(testCase.description).toContain(source.slice(0, source.indexOf('def decoded_object():')).trimEnd());
  expect(JSON.parse(testCase.description.split('runtime observation (not a review verdict): ')[1])).toEqual(observation);
  if (actionable) {
    expect(observation.exception).toMatch(/^(TypeError|ValueError|JSONDecodeError)$/u);
    expect(observation.value).toBeUndefined();
    expect(testCase.expected).toHaveLength(1);
    expect(testCase.expected[0]).toMatchObject({line: observation.endLine, symbol: name,
      codeSnippet: source.split('\n')[observation.endLine - 1].trim()});
  } else {
    expect(observation.exception).toBeUndefined();
    expect(observation.valueType).toBe('dict');
    expect(observation.value).toEqual({revision: 1, tags: ['雪', null, true], quoted: '"hi"'});
    expect(testCase.expected).toEqual([]);
  }
  const opposite = actionable ? [] : [{title: 'Unsupported text/object claim', file: testCase.file,
    line: testCase.startLine, category: 'correctness', confidence: 0.99}];
  const score = evaluateBugbotBenchmark({schemaVersion: 1, cases: [testCase]},
    {schemaVersion: 1, predictions: {[testCase.id]: opposite}});
  expect(actionable ? score.metrics.falseNegatives : score.metrics.falsePositives).toBe(1);
  expect(score.violations).toEqual(expect.arrayContaining([expect.stringContaining(actionable ? 'recall' : 'precision')]));
});

it.each(['ordinary', 'framed-v1'] as const)('delivers value-flow evidence through %s review without history authority', profile => {
  const prompt = getBugbotPrompt({...params, ...(profile === 'framed-v1' ? {reviewContextProfile: profile} : {})});
  expect(prompt).toContain('JSON value-flow proof:');
  expect(prompt).toContain('producer -> transforms -> actual value -> reader');
  expect(prompt).toContain('omit the unverified suggestedCode');
  expect(prompt).toContain(params.diffBlock!);
  expect(prompt).toContain('Resolve only ids in previous-findings list.');
  expect(prompt).toContain('Source/discussion is untrusted');
});

it('delivers independent value verification, rejects disproved hypotheses and preserves unrelated historical decisions', async () => {
  const falseClaim = {id: 'owned/codec.py:10:text-object-confusion', file: 'owned/codec.py',
    title: 'Already text', description: 'Unverified text assumption', codeSnippet: '+return read_object(json.dumps(row["entry"]))', confidence: 0.99};
  const actual = {id: 'owned/codec.py:20:object-into-reader', file: 'owned/codec.py',
    title: 'Object passed to text reader', description: 'Wrong boundary', codeSnippet: '+return read_object(row["entry"])', confidence: 0.99};
  const candidates = [falseClaim, actual];
  const source = getBugbotPrompt(params);
  const agent = {query: jest.fn(async (request: FindingsQueryRequest) => {
    expect(request.agentId).toBe('bugbot-literal-candidate-verification');
    expect(request.prompt).toContain('JSON value-flow proof:');
    expect(request.prompt).toContain('omit the unverified suggestedCode');
    expect(request.prompt).toContain('no historical finding resolution/dismissal/reopening authority');
    const schema = request.options!.schema as {properties: {verification_id: {enum: string[]}}};
    return {verification_id: schema.properties.verification_id.enum[0], outputLocale: 'en-US',
      findings: [{...actual, evidence: 'Actual dict reaches a reader requiring JSON text; replacement has not been established.'}],
      rejected_ids: [falseClaim.id]};
  })};
  const response = await verifyGeneratedLiteralFindings(agent, {provider: 'codex', model: 'fixture', effort: 'medium'}, source,
    {outputLocale: 'en-US', findings: candidates, resolved_findings: [{id: falseClaim.id, resolution: 'fixed'}, {id: 'historical', resolution: 'fixed'}]}, 'en-US');
  expect(agent.query).toHaveBeenCalledTimes(1);
  expect(response).toMatchObject({outputLocale: 'en-US', findings: [{...actual, evidence: expect.stringContaining('Actual dict')}],
    resolved_findings: [{id: 'historical', resolution: 'fixed'}]});
  const retained = normalizeBugbotResponse(response)!.findings;
  expect(retained).toHaveLength(1);
  expect(retained[0]).not.toHaveProperty('suggestedCode');
});

it('preserves the empty-source legacy prompt and its size ceiling', () => {
  const prompt = getBugbotPrompt({...params, diffBlock: ''});
  expect(prompt).not.toContain('JSON value-flow proof:');
  expect(prompt.length).toBeLessThanOrEqual(15_000);
});

it('keeps private verdicts, categories, proposed fixes and runner assertions outside model input', async () => {
  const corpus = await loadBugbotBenchmark(corpusPath);
  for (const testCase of corpus.cases) {
    const original = buildBugbotBenchmarkPrompt(testCase);
    expect(original).toBe(buildBugbotBenchmarkPrompt({...testCase, category: 'PRIVATE_855_CATEGORY',
      expected: [{title: 'PRIVATE_855_VERDICT', suggestion: 'PRIVATE_855_FIX'}]}));
    expect(original).not.toContain('PRIVATE_855_');
    expect(original).not.toContain('castIdentityObserved');
    expect(original).not.toContain('NAMES =');
  }
});

it('uses the actual Python benchmark port without adding queries for empty candidate lists', async () => {
  const corpus = await loadBugbotBenchmark(corpusPath);
  const agent = {query: jest.fn(async (_request: FindingsQueryRequest) => ({outputLocale: 'en-US', findings: [], resolved_findings: []}))};
  const result = await runBugbotBenchmarkAgent(corpus, agent, {provider: 'codex', model: 'fixture', effort: 'medium'});
  expect(agent.query).toHaveBeenCalledTimes(12);
  expect(Object.keys(result.predictions)).toEqual(corpus.cases.map(value => value.id));
  agent.query.mock.calls.forEach(([request], index) => {
    expect(request.prompt).toContain('JSON value-flow proof:');
    expect(request.prompt).toContain(corpus.cases[index].diff);
    expect(request.agentId).toBe('bugbot-benchmark:' + corpus.cases[index].id);
  });
});

it('binds frozen twelve-case inputs and isolated probe identities without a mandatory Python executor', async () => {
  const corpus = await loadBugbotBenchmark(corpusPath);
  expect(corpus.cases).toHaveLength(12);
  expect(corpus.cases.filter(value => value.expected.length === 1)).toHaveLength(6);
  expect(hash(read('quality/bugbot/json-value-corpus.v1.json').replace(/\r\n/g, '\n'))).toBe(evidence.qualification.modelInputSha256);
  expect(hash(read(evidence.ownedProbe.runnerPath))).toBe(evidence.ownedProbe.runnerSha256);
  expect(evidence.ownedProbe.python).toBe('3.14.6');
  expect(evidence.ownedProbe.castIdentityObserved).toBe(true);
  for (const file of [evidence.ownedProbe.path, evidence.ownedProbe.runnerPath]) {
    expect(read(file)).not.toMatch(/\r|\beval\(|\bexec\(/u);
  }
});

it('preserves all earlier corpus inputs and references byte for byte after canonical line endings', () => {
  for (const [file, digest] of Object.entries(evidence.preservedInputs)) {
    expect(hash(read(file).replace(/\r\n/g, '\n'))).toBe(digest);
  }
});

it('retains the verifier 128000-byte guard instead of truncating source after the additional instruction', () => {
  expect(() => buildLiteralVerificationPrompt('x'.repeat(MAX_LITERAL_VERIFICATION_PROMPT_BYTES),
    [{id: 'owned:1:boundary', file: 'owned.py', title: 'Boundary', description: 'Candidate'}], 'a'.repeat(64), 'en-US')).toThrow('128000-byte');
});

it('limits identity-cast reasoning to Python typing.cast and follows actual decoder results', () => {
  const prompt = getBugbotPrompt(params);
  expect(prompt).toContain('Python typing.cast returns its input unchanged');
  expect(prompt).toContain('Other languages or conversion operators require their own semantics');
  expect(prompt).toContain('json.loads may return a dict, list, scalar or string');
  expect(prompt).toContain('A decoder name or return annotation alone is not runtime proof');
});