The Reality of API Integrations
Every Indonesian business I work with uses at least 3-5 external APIs: payment gateways (Midtrans, Xendit), shipping (JNE, J&T, SiCepat), messaging (WhatsApp Business, Twilio), and various SaaS tools.
Here's the reality: these APIs will fail. Not sometimes. Regularly. And when they fail during peak business hours, your entire operation can grind to a halt.
After building 50+ API integrations in Go, here are the patterns that actually work in production.
Pattern 1: The Resilient HTTP Client
Don't use the default http.Client. Build a resilient one:
type ResilientClient struct {
client *http.Client
baseURL string
apiKey string
maxRetries int
logger *zerolog.Logger
}
func NewResilientClient(baseURL, apiKey string) *ResilientClient {
return &ResilientClient{
client: &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{
MaxIdleConns: 100,
MaxIdleConnsPerHost: 10,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
},
},
baseURL: baseURL,
apiKey: apiKey,
maxRetries: 3,
}
}
func (c *ResilientClient) Do(ctx context.Context, method, path string, body interface{}) (*http.Response, error) {
var lastErr error
for attempt := 0; attempt <= c.maxRetries; attempt++ {
if attempt > 0 {
// Exponential backoff with jitter
delay := time.Duration(math.Pow(2, float64(attempt))) * time.Second
jitter := time.Duration(rand.Int63n(int64(delay / 2)))
delay += jitter
c.logger.Warn().Int("attempt", attempt).Dur("delay", delay).Msg("retrying request")
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(delay):
}
}
resp, err := c.execute(ctx, method, path, body)
if err == nil && resp.StatusCode < 500 {
return resp, nil
}
if resp != nil {
resp.Body.Close()
}
lastErr = err
if err == nil {
lastErr = fmt.Errorf("server error: %d", resp.StatusCode)
}
}
return nil, fmt.Errorf("after %d retries: %w", c.maxRetries, lastErr)
}Pattern 2: Circuit Breaker
When an external API is down, stop calling it immediately. Don't waste resources on requests that will fail:
type CircuitBreaker struct {
mu sync.Mutex
state string // closed, open, half-open
failureCount int
successCount int
lastFailureTime time.Time
failureThreshold int
recoveryTimeout time.Duration
}
func (cb *CircuitBreaker) Execute(fn func() error) error {
cb.mu.Lock()
switch cb.state {
case "open":
if time.Since(cb.lastFailureTime) > cb.recoveryTimeout {
cb.state = "half-open"
cb.successCount = 0
} else {
cb.mu.Unlock()
return fmt.Errorf("circuit breaker is open")
}
}
cb.mu.Unlock()
err := fn()
cb.mu.Lock()
defer cb.mu.Unlock()
if err != nil {
cb.failureCount++
cb.lastFailureTime = time.Now()
if cb.failureCount >= cb.failureThreshold {
cb.state = "open"
}
return err
}
if cb.state == "half-open" {
cb.successCount++
if cb.successCount >= 3 {
cb.state = "closed"
cb.failureCount = 0
}
} else {
cb.failureCount = 0
}
return nil
}Pattern 3: Idempotent Operations
Network failures can cause duplicate requests. Make every operation idempotent:
func (s *PaymentService) Charge(ctx context.Context, req PaymentRequest) (*Payment, error) {
// Generate idempotency key
idempotencyKey := generateIdempotencyKey(req.UserID, req.Amount, req.Timestamp)
// Check if we already processed this request
existing, err := s.db.GetPaymentByIdempotencyKey(ctx, idempotencyKey)
if err == nil {
return existing, nil // Already processed
}
// Process with idempotency key
resp, err := s.midtransClient.Charge(ctx, midtrans.ChargeRequest{
TransactionDetails: midtrans.TransactionDetails{
OrderID: idempotencyKey,
GrossAmt: req.Amount,
},
PaymentType: req.PaymentMethod,
IdempotencyKey: idempotencyKey,
})
if err != nil {
return nil, err
}
// Store result
payment := &Payment{
ID: resp.TransactionID,
IdempotencyKey: idempotencyKey,
Status: resp.TransactionStatus,
Amount: req.Amount,
CreatedAt: time.Now(),
}
return payment, s.db.SavePayment(ctx, payment)
}Pattern 4: Timeout Cascading
Every integration needs proper timeout hierarchy:
// Timeout hierarchy: must respect parent timeout
func ProcessOrder(ctx context.Context, order Order) error {
// Parent timeout: 30 seconds total
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
// Step 1: Validate inventory (5 second timeout)
invCtx, invCancel := context.WithTimeout(ctx, 5*time.Second)
defer invCancel()
if err := s.inventory.Validate(invCtx, order.Items); err != nil {
return fmt.Errorf("inventory validation: %w", err)
}
// Step 2: Process payment (10 second timeout)
payCtx, payCancel := context.WithTimeout(ctx, 10*time.Second)
defer payCancel()
payment, err := s.payment.Charge(payCtx, order.Total)
if err != nil {
return fmt.Errorf("payment: %w", err)
}
// Step 3: Reserve inventory (5 second timeout)
resCtx, resCancel := context.WithTimeout(ctx, 5*time.Second)
defer resCancel()
if err := s.inventory.Reserve(resCtx, order.Items); err != nil {
// Payment already charged — need to refund
s.payment.Refund(context.Background(), payment.ID)
return fmt.Errorf("inventory reservation: %w", err)
}
return nil
}Pattern 5: Health Checks and Monitoring
type IntegrationHealth struct {
Name string `json:"name"`
Status string `json:"status"` // healthy, degraded, down
Latency time.Duration `json:"latency"`
LastCheck time.Time `json:"last_check"`
ErrorRate float64 `json:"error_rate"`
CircuitState string `json:"circuit_state"`
}
func (h *HealthChecker) CheckAll(ctx context.Context) []IntegrationHealth {
checks := []IntegrationHealth{}
// Check each integration
integrations := map[string]func(context.Context) error{
"midtrans": s.midtrans.HealthCheck,
"xendit": s.xendit.HealthCheck,
"jne": s.jne.HealthCheck,
"whatsapp": s.whatsapp.HealthCheck,
}
for name, checkFn := range integrations {
start := time.Now()
err := checkFn(ctx)
latency := time.Since(start)
status := "healthy"
if err != nil {
status = "down"
} else if latency > 5*time.Second {
status = "degraded"
}
checks = append(checks, IntegrationHealth{
Name: name,
Status: status,
Latency: latency,
LastCheck: time.Now(),
})
}
return checks
}Real-World Lessons
- Log everything. When a third-party API fails at 2 AM, you need the full request/response to debug. Store at least the last 100 requests per integration.
- Have fallbacks. If Midtrans is down, can you queue the payment and retry later? If WhatsApp Business API fails, fall back to SMS?
- Test failure scenarios. Don't just test the happy path. Inject timeouts, simulate API errors, test your circuit breakers.
- Monitor error rates. A sudden spike in error rates from a third-party API is your early warning system. Alert before your customers notice.
The Bottom Line
API integrations aren't about making a single HTTP call. They're about building a system that degrades gracefully when things go wrong. The patterns above have saved my clients millions of rupiah in avoided downtime and lost transactions.