Statistical significance verified before declaring a winner

Declaring a winner by comparing raw open or click rates without computing a p-value is the statistical equivalent of calling a coin biased after three flips. Even with reasonable sample sizes, peeking at running totals and picking the leader inflates your false positive rate dramatically — a standard 95% confidence threshold applied to a peeked result may provide only 70% actual confidence. You ship the wrong variant and attribute revenue lift to a change that was noise. In iso-25010:2011 terms this is a functional-suitability gap: the experiment infrastructure appears to work but cannot fulfill its core promise of reliable winner selection.

Gate winner selection behind a two-proportion z-test. Throw if the result is not significant rather than returning a soft warning:

function checkSignificance(
  controlConversions: number, controlTotal: number,
  variantConversions: number, variantTotal: number,
  alpha = 0.05
): { pValue: number; isSignificant: boolean; winner: 'control' | 'variant' | 'none' } {
  const p1 = controlConversions / controlTotal
  const p2 = variantConversions / variantTotal
  const pPool = (controlConversions + variantConversions) / (controlTotal + variantTotal)
  const se = Math.sqrt(pPool * (1 - pPool) * (1 / controlTotal + 1 / variantTotal))
  const z = Math.abs(p2 - p1) / se
  const pValue = 2 * (1 - normalCDF(z)) // supply a normalCDF implementation
  const isSignificant = pValue < alpha
  return { pValue, isSignificant, winner: !isSignificant ? 'none' : p2 > p1 ? 'variant' : 'control' }
}

async function declareWinner(experimentId: string, selectedVariant: string) {
  const stats = await getExperimentStats(experimentId)
  const result = checkSignificance(
    stats.control.conversions, stats.control.total,
    stats.variant.conversions, stats.variant.total
  )
  if (!result.isSignificant) {
    throw new Error(`p=${result.pValue.toFixed(4)} — not significant at alpha=0.05`)
  }
  await db.experiments.update({ where: { id: experimentId }, data: { winner: selectedVariant, concluded_at: new Date() } })
}