--- name: customerio-observability-d64d37edee8f description: | Set up Customer.io monitoring and observability. Use when implementing metrics, logging, alerting, or dashboards for Customer.io integrations. Trigger with phrases like "customer.io monitoring", "customer.io metrics", "customer.io dashboard", "customer.io alerts". allowed-tools: Read, Write, Edit, Bash(kubectl:*), Bash(curl:*) version: 1.0.0 license: MIT author: Jeremy Longshore --- # Customer.io Observability ## Overview Implement comprehensive observability for Customer.io integrations including metrics, logging, tracing, and alerting. ## Prerequisites - Customer.io integration deployed - Monitoring infrastructure (Prometheus, Grafana, etc.) - Log aggregation system ## Key Metrics | Metric | Type | Description | |--------|------|-------------| | `customerio_api_latency_ms` | Histogram | API call latency | | `customerio_api_requests_total` | Counter | Total API requests | | `customerio_api_errors_total` | Counter | API error count | | `customerio_email_sent_total` | Counter | Emails sent | | `customerio_email_delivered_total` | Counter | Emails delivered | | `customerio_email_bounced_total` | Counter | Email bounces | | `customerio_webhook_received_total` | Counter | Webhooks received | ## Instructions ### Step 1: Metrics Collection ```typescript // lib/metrics.ts import { Counter, Histogram, Registry } from 'prom-client'; const register = new Registry(); // API metrics export const apiLatency = new Histogram({ name: 'customerio_api_latency_ms', help: 'Customer.io API call latency in milliseconds', labelNames: ['operation', 'status'], buckets: [10, 25, 50, 100, 250, 500, 1000, 2500, 5000], registers: [register] }); export const apiRequests = new Counter({ name: 'customerio_api_requests_total', help: 'Total Customer.io API requests', labelNames: ['operation', 'status'], registers: [register] }); export const apiErrors = new Counter({ name: 'customerio_api_errors_total', help: 'Total Customer.io API errors', labelNames: ['operation', 'error_type'], registers: [register] }); // Email metrics export const emailsSent = new Counter({ name: 'customerio_email_sent_total', help: 'Total emails sent via Customer.io', labelNames: ['campaign_type'], registers: [register] }); export const emailsDelivered = new Counter({ name: 'customerio_email_delivered_total', help: 'Total emails delivered', labelNames: ['campaign_type'], registers: [register] }); export const emailsBounced = new Counter({ name: 'customerio_email_bounced_total', help: 'Total email bounces', labelNames: ['bounce_type'], registers: [register] }); // Webhook metrics export const webhooksReceived = new Counter({ name: 'customerio_webhook_received_total', help: 'Total webhooks received from Customer.io', labelNames: ['event_type'], registers: [register] }); export { register }; ``` ### Step 2: Instrumented Client ```typescript // lib/customerio-instrumented.ts import { TrackClient, RegionUS } from '@customerio/track'; import * as metrics from './metrics'; export class InstrumentedCustomerIO { private client: TrackClient; constructor(siteId: string, apiKey: string) { this.client = new TrackClient(siteId, apiKey, { region: RegionUS }); } async identify(userId: string, attributes: Record): Promise { const timer = metrics.apiLatency.startTimer({ operation: 'identify' }); try { await this.client.identify(userId, attributes); timer({ status: 'success' }); metrics.apiRequests.inc({ operation: 'identify', status: 'success' }); } catch (error: any) { timer({ status: 'error' }); metrics.apiRequests.inc({ operation: 'identify', status: 'error' }); metrics.apiErrors.inc({ operation: 'identify', error_type: error.statusCode || 'unknown' }); throw error; } } async track(userId: string, event: string, data?: Record): Promise { const timer = metrics.apiLatency.startTimer({ operation: 'track' }); try { await this.client.track(userId, { name: event, data }); timer({ status: 'success' }); metrics.apiRequests.inc({ operation: 'track', status: 'success' }); } catch (error: any) { timer({ status: 'error' }); metrics.apiRequests.inc({ operation: 'track', status: 'error' }); metrics.apiErrors.inc({ operation: 'track', error_type: error.statusCode || 'unknown' }); throw error; } } } ``` ### Step 3: Structured Logging ```typescript // lib/logger.ts import pino from 'pino'; export const logger = pino({ name: 'customerio', level: process.env.LOG_LEVEL || 'info', formatters: { level: (label) => ({ level: label }) }, base: { service: 'customerio-integration', environment: process.env.NODE_ENV } }); // Logging wrapper for Customer.io operations export function logOperation( operation: string, userId: string, data: any, result: 'success' | 'error', error?: Error ) { const logData = { operation, userId, result, data: sanitizeForLogging(data), ...(error && { error: { message: error.message, stack: error.stack } }) }; if (result === 'error') { logger.error(logData, `Customer.io ${operation} failed`); } else { logger.info(logData, `Customer.io ${operation} succeeded`); } } // Remove PII from logs function sanitizeForLogging(data: any): any { if (!data) return data; const sanitized = { ...data }; const piiFields = ['email', 'phone', 'address', 'ssn']; for (const field of piiFields) { if (sanitized[field]) { sanitized[field] = '[REDACTED]'; } } return sanitized; } ``` ### Step 4: Distributed Tracing ```typescript // lib/tracing.ts import { trace, SpanKind, SpanStatusCode } from '@opentelemetry/api'; const tracer = trace.getTracer('customerio-integration'); export async function withTracing( operationName: string, attributes: Record, operation: () => Promise ): Promise { return tracer.startActiveSpan( `customerio.${operationName}`, { kind: SpanKind.CLIENT, attributes: { 'customerio.operation': operationName, ...attributes } }, async (span) => { try { const result = await operation(); span.setStatus({ code: SpanStatusCode.OK }); return result; } catch (error: any) { span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); span.recordException(error); throw error; } finally { span.end(); } } ); } // Usage await withTracing('identify', { userId }, () => client.identify(userId, attributes) ); ``` ### Step 5: Grafana Dashboard ```json { "dashboard": { "title": "Customer.io Integration", "panels": [ { "title": "API Latency (p50, p95, p99)", "type": "timeseries", "targets": [ { "expr": "histogram_quantile(0.50, rate(customerio_api_latency_ms_bucket[5m]))", "legendFormat": "p50" }, { "expr": "histogram_quantile(0.95, rate(customerio_api_latency_ms_bucket[5m]))", "legendFormat": "p95" }, { "expr": "histogram_quantile(0.99, rate(customerio_api_latency_ms_bucket[5m]))", "legendFormat": "p99" } ] }, { "title": "API Request Rate", "type": "timeseries", "targets": [ { "expr": "rate(customerio_api_requests_total[5m])", "legendFormat": "{{operation}} - {{status}}" } ] }, { "title": "Error Rate", "type": "stat", "targets": [ { "expr": "sum(rate(customerio_api_errors_total[5m])) / sum(rate(customerio_api_requests_total[5m])) * 100", "legendFormat": "Error Rate %" } ], "fieldConfig": { "defaults": { "thresholds": { "steps": [ { "value": 0, "color": "green" }, { "value": 1, "color": "yellow" }, { "value": 5, "color": "red" } ] } } } }, { "title": "Email Delivery Funnel", "type": "bargauge", "targets": [ { "expr": "sum(customerio_email_sent_total)", "legendFormat": "Sent" }, { "expr": "sum(customerio_email_delivered_total)", "legendFormat": "Delivered" }, { "expr": "sum(customerio_email_bounced_total)", "legendFormat": "Bounced" } ] } ] } } ``` ### Step 6: Alerting Rules ```yaml # prometheus/alerts/customerio.yml groups: - name: customerio rules: - alert: CustomerIOHighErrorRate expr: | sum(rate(customerio_api_errors_total[5m])) / sum(rate(customerio_api_requests_total[5m])) > 0.05 for: 5m labels: severity: critical annotations: summary: Customer.io API error rate > 5% description: Error rate is {{ $value | printf "%.2f" }}% - alert: CustomerIOHighLatency expr: | histogram_quantile(0.99, rate(customerio_api_latency_ms_bucket[5m])) > 5000 for: 10m labels: severity: warning annotations: summary: Customer.io p99 latency > 5s description: p99 latency is {{ $value | printf "%.0f" }}ms - alert: CustomerIOHighBounceRate expr: | sum(rate(customerio_email_bounced_total[1h])) / sum(rate(customerio_email_sent_total[1h])) > 0.05 for: 30m labels: severity: warning annotations: summary: Email bounce rate > 5% description: Bounce rate is {{ $value | printf "%.2f" }}% - alert: CustomerIOWebhookProcessingFailed expr: | sum(rate(customerio_webhook_errors_total[5m])) > 0 for: 5m labels: severity: critical annotations: summary: Customer.io webhook processing failures description: {{ $value }} webhooks failed in last 5 minutes ``` ## Observability Checklist - [ ] API latency metrics collected - [ ] Error rate tracking enabled - [ ] Structured logging implemented - [ ] Distributed tracing configured - [ ] Grafana dashboard created - [ ] Alert rules defined - [ ] PII redacted from logs - [ ] Log retention policy set ## Error Handling | Issue | Solution | |-------|----------| | Missing metrics | Check metric registration | | High cardinality | Reduce label values | | Log volume too high | Adjust log level | ## Resources - [Prometheus Best Practices](https://prometheus.io/docs/practices/) - [OpenTelemetry Node.js](https://opentelemetry.io/docs/instrumentation/js/) ## Next Steps After observability setup, proceed to `customerio-advanced-troubleshooting` for debugging. --- Source attribution: Dicklesworthstone Original project: https://github.com/Dicklesworthstone/pi_agent_rust/blob/f28fc46ee0db8e710d95b8b55592cd38d60f2b26/tests/ext_conformance/artifacts/plugins-community/plugins/saas-packs/customerio-pack/skills/customerio-observability/SKILL.md License: MIT License information: https://opensource.org/license/mit Original upstream instructions; downloaded from Mega AI Hub. ## Upstream license notice MIT License (with OpenAI/Anthropic Rider) Copyright (c) 2026 Jeffrey Emanuel Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: ADDITIONAL RIDER / RESTRICTION (OpenAI / Anthropic): This rider is part of the "conditions" of this License. In the event of any conflict between this rider and any other portion of this License, this rider controls. "Restricted Parties" means OpenAI, L.L.C.; Anthropic, PBC; any of their respective Affiliates; and any person or entity acting directly or indirectly on behalf of, for the benefit of, or under the direction of any of the foregoing (including any officer, director, employee, contractor, agent, consultant, service provider, or representative). Notwithstanding any other provision of this License, no rights are granted to any Restricted Party. Any purported license, sublicense, assignment, transfer, or other permission to any Restricted Party is null and void absent the express prior written permission of Jeffrey Emanuel. You may not provide, disclose, distribute, sublicense, sell, lease, lend, host, make available, or otherwise permit access to the Software or any derivative work of the Software (as defined in applicable copyright law) (collectively, "Derivative Works") to or for any Restricted Party. For purposes of this rider, "use" includes, without limitation: copying, modifying, merging, publishing, distributing, sublicensing, selling, transferring, making available, hosting, deploying, executing, benchmarking, testing, analyzing, indexing, or incorporating the Software or any Derivative Works into any dataset, training corpus, evaluation harness, or pipeline for machine learning or other automated systems. This rider applies to the Software and all Derivative Works. As a condition of use, you agree that this rider is a precondition to exercising any rights under this License, and you agree that any distribution of the Software or any Derivative Works must include this rider provision unmodified. Any breach of this rider automatically and immediately terminates the permissions granted by this License. Upon termination, you must immediately cease all use and distribution of the Software and any Derivative Works and destroy all copies under your control. You agree that a breach of this rider would cause irreparable harm and that Jeffrey Emanuel may seek injunctive or other equitable relief to enforce this rider, in addition to any other remedies available at law. To the maximum extent permitted by applicable law, the prevailing party in any action to enforce this rider shall be entitled to recover reasonable attorneys' fees and costs. For purposes of this rider, "Affiliate" means any entity that directly or indirectly controls, is controlled by, or is under common control with the specified party. "Control" means ownership of more than 50% of the voting securities or other ownership interest, or the power to direct management or policies by contract or otherwise. The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # Mega AI Hub resource metadata Creator / publisher: Dicklesworthstone Original project: https://github.com/Dicklesworthstone/pi_agent_rust/blob/f28fc46ee0db8e710d95b8b55592cd38d60f2b26/tests/ext_conformance/artifacts/plugins-community/plugins/saas-packs/customerio-pack/skills/customerio-observability/SKILL.md Attribution basis: Source repository record Fields of work: Marketing, Sales Roles: Customer Success Manager, Growth Marketer, Product Marketer Specialties: API Tools: Grafana, Prometheus Country applicability: Country not specified Classification basis: Source catalog labels Instruction SHA-256: ebb0f582bc59f953c86c5d14bc1adf71dd64e290faba4742d47bbaf64d984381