Belajar QA Automation Engineer - Testing AI/LLM Applications

AI Testing Challenges:
├── Non-deterministic outputs (beda setiap run)
├── Hallucination (AI mengarang jawaban)
├── Quality subjectif (tidak ada pass/fail jelas)
├── Latency tinggi (response time bervariasi)
├── Cost per test (API calls mahal)
└── Evaluation metrics tidak standar

// Test prompt handling
test('prompt validation', async ({ request }) => {
  // Test empty prompt
  const emptyResponse = await request.post('/api/chat', {
    data: { message: '' },
  });
  expect(emptyResponse.status()).toBe(400);
 
  // Test too long prompt
  const longResponse = await request.post('/api/chat', {
    data: { message: 'A'.repeat(100000) },
  });
  expect(longResponse.status()).toBe(400);
 
  // Test valid prompt
  const validResponse = await request.post('/api/chat', {
    data: { message: 'Hello, how are you?' },
  });
  expect(validResponse.ok()).toBeTruthy();
});

// Evaluate AI output quality
async function evaluateAIOutput(input: string, output: string) {
  const metrics = {
    // Relevance: apakah output menjawab pertanyaan?
    relevance: await checkRelevance(input, output),
 
    // Accuracy: apakah output factually correct?
    accuracy: await checkAccuracy(output),
 
    // Coherence: apakah output logical dan terstruktur?
    coherence: await checkCoherence(output),
 
    // Safety: apakah output tidak harmful?
    safety: await checkSafety(output),
 
    // Length: apakah output terlalu panjang/pendek?
    length: output.length > 10 && output.length < 5000,
  };
 
  return metrics;
}

// Non-deterministic assertion
test('AI response quality', async ({ request }) => {
  const response = await request.post('/api/chat', {
    data: { message: 'What is 2+2?' },
  });
 
  const data = await response.json();
 
  // Pattern matching, bukan exact match
  expect(data.response).toMatch(/\b4\b/);
  expect(data.response.length).toBeGreaterThan(10);
  expect(data.response.toLowerCase()).toContain('four');
});

// Detect hallucination
test('no hallucination', async ({ request }) => {
  const response = await request.post('/api/chat', {
    data: { message: 'Who is the president of Indonesia in 2026?' },
  });
 
  const data = await response.json();
 
  // Check against known facts
  const knownFacts = await getKnownFacts('indonesia-president-2026');
  const containsFact = knownFacts.some(fact =>
    data.response.toLowerCase().includes(fact.toLowerCase())
  );
 
  // Atau: verify dengan web search
  const verified = await verifyWithWebSearch(data.response);
  expect(verified).toBeTruthy();
});

AI Regression Testing:
├── Golden dataset: simpan input/output pairs
├── Semantic comparison: bukan exact match
├── Quality threshold: minimum quality score
├── Cost monitoring: track API costs
└── Performance baseline: response time thresholds

test.describe('Chatbot Testing', () => {
  test('responds to greeting', async ({ request }) => {
    const response = await request.post('/api/chat', {
      data: { message: 'Hello' },
    });
    const data = await response.json();
    expect(data.response.toLowerCase()).toMatch(/hello|hi|hey/);
  });
 
  test('handles out-of-scope question', async ({ request }) => {
    const response = await request.post('/api/chat', {
      data: { message: 'What is the meaning of life?' },
    });
    const data = await response.json();
    expect(data.response.length).toBeGreaterThan(0);
  });
});