New Relic APM 示例

// New Relic Node.js Agent Configuration
// Install: npm install newrelic

// newrelic.js - Agent configuration file
'use strict'

exports.config = {
  // Application name (appears in New Relic UI)
  app_name: ['My Web App'],
  // License key from New Relic account
  license_key: process.env.NEW_RELIC_LICENSE_KEY || 'your-license-key-here',
  // Logging configuration
  logging: {
    level: 'info',
    filepath: '/var/log/newrelic/nr-agent.log',
    enabled: true
  },
  // Distributed tracing
  distributed_tracing: {
    enabled: true
  },
  // Application logging with context
  application_logging: {
    enabled: true,
    forwarding: {
      enabled: true,
      max_samples_stored: 10000
    },
    metrics: {
      enabled: true
    },
    local_decorating: {
      enabled: true
    }
  },
  // Browser monitoring
  browser_monitoring: {
    enable: true
  },
  // Error collection
  error_collector: {
    enabled: true,
    capture_source_map: true
  },
  // Transaction traces
  transaction_tracer: {
    enabled: true,
    transaction_threshold: 0.5, // seconds
    record_sql: 'obfuscated',
    stack_trace_threshold: 0.5
  },
  // Custom insights events
  custom_insights_events: {
    enabled: true
  },
  // Custom metrics
  custom_metrics_enabled: true,
  // API configuration
  apdex_t: 0.5, // Apdex threshold in seconds
  // Browser monitoring
  browser_monitoring: {
    enable: true
  }
}

// Application entry point - require newrelic first
require('newrelic')

const express = require('express')
const app = express()
const logger = require('winston')

// Create custom New Relic API instance
const newrelic = require('newrelic')

// Custom transaction example
app.get('/api/users/:id', async (req, res) => {
  const userId = req.params.id

  // Add custom attributes to current transaction
  newrelic.addCustomAttribute('userId', userId)
  newrelic.addCustomAttribute('userRole', 'customer')

  try {
    // Segment for user lookup
    const user = await newrelic.startSegment('user-lookup', true, async () => {
      return await database.findUserById(userId)
    })

    if (!user) {
      newrelic.noticeError(new Error('User not found'), { userId })
      return res.status(404).json({ error: 'User not found' })
    }

    // Record custom event
    newrelic.recordCustomEvent('UserAccess', {
      userId: user.id,
      accountType: user.type,
      accessTime: Date.now()
    })

    // Segment for related data
    const userOrders = await newrelic.startSegment('user-orders-fetch', true, async () => {
      return await database.getUserOrders(userId)
    })

    res.json({
      user: user,
      orderCount: userOrders.length
    })

  } catch (error) {
    // Record error with context
    newrelic.noticeError(error, {
      userId: userId,
      endpoint: '/api/users/:id',
      method: 'GET'
    })
    res.status(500).json({ error: 'Internal server error' })
  }
})

// Background job with custom transaction
async function processEmailQueue() {
  return newrelic.startBackgroundTransaction('process-email-queue', 'background-job', async () => {
    const transaction = newrelic.getTransaction()

    try {
      const emails = await getEmailQueue()
      transaction.addCustomAttribute('emailCount', emails.length)

      for (const email of emails) {
        await newrelic.startSegment('send-email', true, async () => {
          await emailService.send(email)
          transaction.recordMetric('Custom/Emails/Sent', 1)
        })
      }

    } catch (error) {
      newrelic.noticeError(error)
      transaction.recordMetric('Custom/Emails/Failed', 1)
      throw error
    }
  })
}

// Custom metrics recording
function recordBusinessMetrics() {
  const activeUsers = getActiveUserCount()
  const cartValue = getAverageCartValue()

  newrelic.recordMetric('Custom/Business/ActiveUsers', activeUsers)
  newrelic.recordMetric('Custom/Business/AverageCartValue', cartValue)
  newrelic.recordMetric('Custom/Business/OrdersPerMinute', getOrdersPerMinute())
}

// Performance monitoring middleware
app.use((req, res, next) => {
  const start = Date.now()

  res.on('finish', () => {
    const duration = Date.now() - start

    // Record custom response time metric
    newrelic.recordMetric('Custom/API/ResponseTime', duration / 1000)

    // Add response attributes
    newrelic.addCustomAttribute('httpMethod', req.method)
    newrelic.addCustomAttribute('endpoint', req.route?.path || req.path)
    newrelic.addCustomAttribute('statusCode', res.statusCode)

    // Record endpoint-specific metrics
    if (req.route?.path) {
      newrelic.recordMetric(`Custom/Endpoint/${req.method}_${req.route.path}_ResponseTime`, duration / 1000)
    }
  })

  next()
})

// Health check endpoint
app.get('/health', (req, res) => {
  const health = {
    status: 'healthy',
    timestamp: new Date().toISOString(),
    uptime: process.uptime(),
    memory: process.memoryUsage()
  }

  // Add health metrics
  newrelic.recordMetric('Custom/Health/Uptime', process.uptime())
  newrelic.recordMetric('Custom/Health/MemoryUsage', process.memoryUsage().heapUsed / 1024 / 1024)

  res.json(health)
})

// Error handling middleware
app.use((error, req, res, next) => {
  // Record error in New Relic
  newrelic.noticeError(error, {
    endpoint: req.path,
    method: req.method,
    userAgent: req.get('User-Agent'),
    ip: req.ip
  })

  // Record error metric
  newrelic.incrementMetric('Custom/Errors/Unhandled')

  res.status(500).json({
    error: 'Internal server error',
    requestId: req.id
  })
})

// Schedule periodic metrics
setInterval(recordBusinessMetrics, 60000) // Every minute

const PORT = process.env.PORT || 3000
app.listen(PORT, () => {
  console.log(`Server running on port ${PORT}`)

  // Record deployment event
  newrelic.recordDeploymentEvent({
    deployId: process.env.DEPLOY_ID || 'local-dev',
    deployDescription: 'Application started',
    user: process.env.USER || 'system',
    changelog: process.env.CHANGELOG || 'Initial deployment'
  })
})

// Graceful shutdown
process.on('SIGTERM', () => {
  newrelic.startSegment('shutdown', false, () => {
    newrelic.recordCustomEvent('ApplicationShutdown', {
      uptime: process.uptime(),
      timestamp: Date.now()
    })
    process.exit(0)
  })
})

# New Relic Python Agent Configuration
# Install: pip install newrelic

# newrelic.ini - Agent configuration file
newrelic_ini = '''
[newrelic]
# Application name (appears in New Relic UI)
app_name = My Python Application
license_key = your-license-key-here

# Environment
environment = production

# Logging
log_level = info
log_file = /var/log/newrelic/newrelic-python.log

# Monitor mode
monitor_mode = true

# High security mode (restricts data collection)
high_security = false

# Browser monitoring
browser_monitoring.auto_instrument = true

# Background tasks
enabled_background_task = true

# Transaction traces
transaction_tracer.enabled = true
transaction_tracer.transaction_threshold = 0.5
transaction_tracer.record_sql = obfuscated
transaction_tracer.stack_trace_threshold = 0.5

# Error collector
error_collector.enabled = true
error_collector.capture_source = true

# Browser monitoring
browser_monitoring.auto_instrument = true

# Custom metrics
custom_metrics.enabled = true

# Custom events
custom_insights_events.enabled = true

# Distributed tracing
distributed_tracing.enabled = true

# Application logging
application_logging.enabled = true
application_logging.forwarding.enabled = true
application_logging.metrics.enabled = true
application_logging.local_decorating.enabled = true
'''

# Django example application
# settings.py
INSTALLED_APPS = [
    # ... other apps
    'newrelic',
]

# Middleware configuration
MIDDLEWARE = [
    'newrelic.agent.NewRelicMiddleware',
    # ... other middleware
]

# views.py
import newrelic
from django.http import JsonResponse
from django.views.decorators.http import require_http_methods
import time

@require_http_methods(["GET"])
def user_profile(request, user_id):
    # Add custom attributes
    newrelic.agent.add_custom_parameter('user_id', user_id)
    newrelic.agent.add_custom_parameter('endpoint', '/user/profile')

    try:
        # Custom segment for database query
        with newrelic.agent.FunctionTrace('user_lookup', 'database'):
            user = User.objects.get(id=user_id)

        # Add more context
        newrelic.agent.add_custom_parameter('user_type', user.user_type)
        newrelic.agent.add_custom_parameter('account_age_days',
                                         (timezone.now() - user.date_joined).days)

        # Record custom event
        newrelic.agent.record_custom_event('UserProfileView', {
            'user_id': user_id,
            'user_type': user.user_type,
            'view_timestamp': time.time()
        })

        return JsonResponse({
            'user': {
                'id': user.id,
                'name': user.get_full_name(),
                'email': user.email,
                'type': user.user_type
            }
        })

    except User.DoesNotExist:
        # Record notice error
        newrelic.agent.notice_error(Exception(f"User {user_id} not found"))
        newrelic.agent.record_custom_event('UserNotFound', {
            'user_id': user_id,
            'endpoint': '/user/profile'
        })
        return JsonResponse({'error': 'User not found'}, status=404)

    except Exception as e:
        # Record error with context
        newrelic.agent.notice_error(e, {
            'user_id': user_id,
            'endpoint': '/user/profile',
            'error_type': type(e).__name__
        })
        return JsonResponse({'error': 'Internal server error'}, status=500)

# Flask example application
# app.py
from flask import Flask, request, jsonify
import newrelic.agent

app = Flask(__name__)

@app.route('/api/orders', methods=['GET', 'POST'])
def handle_orders():
    # Add custom attributes
    newrelic.agent.add_custom_parameter('http_method', request.method)
    newrelic.agent.add_custom_parameter('endpoint', '/api/orders')

    if request.method == 'GET':
        return get_orders()
    elif request.method == 'POST':
        return create_order()

def get_orders():
    # Custom segment for order retrieval
    with newrelic.agent.FunctionTrace('order_retrieval', 'database'):
        orders = Order.query.filter_by(user_id=request.args.get('user_id')).all()

    # Record metrics
    newrelic.agent.record_metric('Custom/Orders/Retrieved', len(orders))
    newrelic.agent.add_custom_parameter('order_count', len(orders))

    return jsonify({
        'orders': [order.to_dict() for order in orders]
    })

def create_order():
    order_data = request.get_json()

    # Custom segment for order creation
    with newrelic.agent.FunctionTrace('order_creation', 'business'):
        order = Order.create(order_data)

        # Process payment
        with newrelic.agent.FunctionTrace('payment_processing', 'external'):
            payment_result = process_payment(order)

    # Record business metrics
    newrelic.agent.record_metric('Custom/Orders/Created', 1)
    newrelic.agent.record_metric('Custom/Orders/Value', order.total_amount)
    newrelic.agent.add_custom_parameter('order_value', order.total_amount)
    newrelic.agent.add_custom_parameter('payment_method', order_data.get('payment_method'))

    # Record custom event
    newrelic.agent.record_custom_event('OrderCreated', {
        'order_id': order.id,
        'user_id': order.user_id,
        'total_amount': order.total_amount,
        'payment_method': order_data.get('payment_method'),
        'timestamp': time.time()
    })

    return jsonify({
        'order_id': order.id,
        'status': 'created',
        'total_amount': order.total_amount
    }), 201

# Background task example with Celery
# tasks.py
from celery import Celery
import newrelic.agent

celery_app = Celery('tasks')

@celery_app.task
def process_email_notification(user_id, message):
    # Start background transaction
    newrelic.agent.set_transaction_name('process_email_notification', 'Background')

    try:
        # Add context
        newrelic.agent.add_custom_parameter('user_id', user_id)
        newrelic.agent.add_custom_parameter('message_type', message['type'])

        # Fetch user data
        with newrelic.agent.FunctionTrace('fetch_user', 'database'):
            user = User.objects.get(id=user_id)

        # Send email
        with newrelic.agent.FunctionTrace('send_email', 'external'):
            email_result = send_email(user.email, message)

        # Record success
        newrelic.agent.record_metric('Custom/Emails/Sent', 1)
        newrelic.agent.record_custom_event('EmailNotificationSent', {
            'user_id': user_id,
            'message_type': message['type'],
            'timestamp': time.time()
        })

        return {'status': 'success', 'email': user.email}

    except Exception as e:
        # Record error
        newrelic.agent.notice_error(e, {
            'user_id': user_id,
            'task': 'process_email_notification'
        })
        newrelic.agent.record_metric('Custom/Emails/Failed', 1)
        raise

# Custom metrics function
def record_application_metrics():
    """Record periodic business metrics"""
    import redis
    from django.db import connection

    # Active users (from Redis)
    redis_client = redis.Redis()
    active_users = len(redis_client.keys('user:active:*'))
    newrelic.agent.record_metric('Custom/Users/Active', active_users)

    # Database connection count
    with newrelic.agent.FunctionTrace('db_connection_check', 'system'):
        db_connections = len(connection.queries)
        newrelic.agent.record_metric('Custom/Database/Connections', db_connections)

    # Average response time
    avg_response_time = calculate_avg_response_time()
    newrelic.agent.record_metric('Custom/Performance/AverageResponseTime', avg_response_time)

    # Queue length
    queue_length = celery_app.control.inspect().active()
    if queue_length:
        total_tasks = sum(len(tasks) for tasks in queue_length.values())
        newrelic.agent.record_metric('Custom/Queue/Length', total_tasks)

# Schedule periodic metrics recording
import schedule
import threading

def run_periodic_metrics():
    record_application_metrics()
    schedule.every(60).seconds.do(record_application_metrics)

def schedule_thread():
    while True:
        schedule.run_pending()
        time.sleep(1)

thread = threading.Thread(target=schedule_thread)
thread.daemon = True
thread.start()

# Middleware for additional instrumentation
class CustomNewRelicMiddleware:
    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        # Add request attributes
        newrelic.agent.add_custom_parameter('user_agent',
                                         request.META.get('HTTP_USER_AGENT', 'Unknown'))
        newrelic.agent.add_custom_parameter('remote_addr',
                                         request.META.get('REMOTE_ADDR'))

        # Add authentication status
        if hasattr(request, 'user') and request.user.is_authenticated:
            newrelic.agent.add_custom_parameter('user_authenticated', True)
            newrelic.agent.add_custom_parameter('user_id', request.user.id)
        else:
            newrelic.agent.add_custom_parameter('user_authenticated', False)

        response = self.get_response(request)

        # Record response metrics
        newrelic.agent.record_metric('Custom/Requests/Count', 1)
        newrelic.agent.add_custom_parameter('response_status', response.status_code)

        return response

# deployment script
# deploy.py
import newrelic.agent
import os

def record_deployment():
    """Record deployment event in New Relic"""
    newrelic.agent.record_deployment_event(
        deploy_id=os.getenv('DEPLOY_ID', 'local-dev'),
        deploy_description=os.getenv('DEPLOY_DESCRIPTION', 'Manual deployment'),
        deploy_user=os.getenv('DEPLOY_USER', os.getenv('USER', 'system')),
        changelog=os.getenv('CHANGELOG', 'Deployment via script'),
        timestamp=time.time()
    )

if __name__ == '__main__':
    record_deployment()
    print("Deployment recorded in New Relic")

{
  "name": "Application Performance Dashboard",
  "description": "Real-time monitoring of application performance and business metrics",
  "pages": [
    {
      "name": "Overview",
      "description": "Key performance indicators and health status",
      "widgets": [
        {
          "title": "Request Rate",
          "visualization": "billboard",
          "nrql": "SELECT count(apm.service.transaction.duration) FROM Metric WHERE appName = 'My Web App' TIMESERIES 1 minute",
          "row": 1,
          "column": 1,
          "width": 3,
          "height": 3
        },
        {
          "title": "Average Response Time",
          "visualization": "line",
          "nrql": "SELECT percentile(apm.service.transaction.duration, 50, 95, 99) FROM Metric WHERE appName = 'My Web App' TIMESERIES 5 minutes",
          "row": 1,
          "column": 4,
          "width": 6,
          "height": 3
        },
        {
          "title": "Error Rate",
          "visualization": "billboard",
          "nrql": "SELECT percentage(count(*), WHERE apm.service.transaction.error IS true) FROM Metric WHERE appName = 'My Web App' AND metricTimesliceName IN ('apm.service.transaction.duration') SINCE 1 hour ago",
          "row": 1,
          "column": 10,
          "width": 3,
          "height": 3
        },
        {
          "title": "Apdex Score",
          "visualization": "gauge",
          "nrql": "SELECT apdex(apm.service.apdex) FROM Metric WHERE appName = 'My Web App' SINCE 1 hour ago",
          "row": 4,
          "column": 1,
          "width": 3,
          "height": 3
        },
        {
          "title": "Transaction Breakdown",
          "visualization": "pie",
          "nrql": "SELECT count(apm.service.transaction.duration) FROM Metric WHERE appName = 'My Web App' FACET transactionType SINCE 1 hour ago",
          "row": 4,
          "column": 4,
          "width": 4,
          "height": 3
        },
        {
          "title": "Top Slow Transactions",
          "visualization": "table",
          "nrql": " SELECT count(apm.service.transaction.duration), percentile(apm.service.transaction.duration, 95), appName FROM Metric WHERE appName = 'My Web App' FACET name, appName LIMIT 10 SINCE 1 hour ago",
          "row": 4,
          "column": 8,
          "width": 5,
          "height": 3
        }
      ]
    },
    {
      "name": "Infrastructure",
      "description": "System resources and infrastructure health",
      "widgets": [
        {
          "title": "CPU Usage",
          "visualization": "line",
          "nrql": " SELECT average(cpuPercent) FROM SystemSample SINCE 1 hour ago FACET hostname TIMESERIES 1 minute",
          "row": 1,
          "column": 1,
          "width": 6,
          "height": 3
        },
        {
          "title": "Memory Usage",
          "visualization": "line",
          "nrql": " SELECT average(memoryUsedPercent) FROM SystemSample SINCE 1 hour ago FACET hostname TIMESERIES 1 minute",
          "row": 1,
          "column": 7,
          "width": 6,
          "height": 3
        },
        {
          "title": "Disk I/O",
          "visualization": "area",
          "nrql": " SELECT average(diskIOPercent) FROM SystemSample SINCE 1 hour ago FACET device TIMESERIES 1 minute",
          "row": 4,
          "column": 1,
          "width": 6,
          "height": 3
        },
        {
          "title": "Network Traffic",
          "visualization": "line",
          "nrql": " SELECT average(networkRxBytesPerSecond), average(networkTxBytesPerSecond) FROM SystemSample SINCE 1 hour ago FACET hostname TIMESERIES 1 minute",
          "row": 4,
          "column": 7,
          "width": 6,
          "height": 3
        }
      ]
    },
    {
      "name": "Business Metrics",
      "description": "Application-specific business KPIs and metrics",
      "widgets": [
        {
          "title": "Active Users",
          "visualization": "line",
          "nrql": " SELECT latest(Custom/Users/Active) FROM Metric SINCE 24 hours ago TIMESERIES 15 minutes",
          "row": 1,
          "column": 1,
          "width": 6,
          "height": 3
        },
        {
          "title": "Order Value",
          "visualization": "line",
          "nrql": " SELECT sum(Custom/Orders/Value) FROM Metric SINCE 24 hours ago TIMESERIES 1 hour COMPARE WITH 1 day ago",
          "row": 1,
          "column": 7,
          "width": 6,
          "height": 3
        },
        {
          "title": "Orders by Payment Method",
          "visualization": "pie",
          "nrql": " SELECT count(*) FROM OrderCreated FACET paymentMethod SINCE 24 hours ago",
          "row": 4,
          "column": 1,
          "width": 4,
          "height": 3
        },
        {
          "title": "Conversion Funnel",
          "visualization": "funnel",
          "nrql": " SELECT count(*) FROM PageView WHERE appName = 'My Web App' AND pageUrl LIKE '%/product/%' SINCE 24 hours ago COMPARE WITH 1 week ago",
          "row": 4,
          "column": 5,
          "width": 4,
          "height": 3
        },
        {
          "title": "Revenue Trend",
          "visualization": "area",
          "nrql": " SELECT sum(totalAmount) FROM OrderCreated SINCE 30 days ago TIMESERIES 1 day",
          "row": 4,
          "column": 9,
          "width": 4,
          "height": 3
        }
      ]
    }
  ],
  "permissions": "PUBLIC_READ_WRITE"
}

# Terraform configuration for New Relic dashboard
# main.tf
terraform {
  required_providers {
    newrelic = {
      source  = "newrelic/newrelic"
      version = "~> 2.0"
    }
  }
}

provider "newrelic" {
  api_key    = var.newrelic_api_key
  account_id = var.newrelic_account_id
  region     = "US" # or "EU"
}

variable "newrelic_api_key" {
  description = "New Relic API key"
  type        = string
  sensitive   = true
}

variable "newrelic_account_id" {
  description = "New Relic account ID"
  type        = string
}

resource "newrelic_dashboard" "application_dashboard" {
  title = "Application Performance Dashboard"

  widget {
    title         = "Request Rate"
    visualization = "billboard"
    row           = 1
    column        = 1
    width         = 3
    height        = 3

    nrql_query {
      query = "SELECT count(apm.service.transaction.duration) FROM Metric WHERE appName = 'My Web App' TIMESERIES 1 minute"
    }
  }

  widget {
    title         = "Average Response Time"
    visualization = "line"
    row           = 1
    column        = 4
    width         = 6
    height        = 3

    nrql_query {
      query = "SELECT percentile(apm.service.transaction.duration, 50, 95, 99) FROM Metric WHERE appName = 'My Web App' TIMESERIES 5 minutes"
    }
  }

  widget {
    title         = "Error Rate"
    visualization = "billboard"
    row           = 1
    column        = 10
    width         = 3
    height        = 3

    nrql_query {
      query = "SELECT percentage(count(*), WHERE apm.service.transaction.error IS true) FROM Metric WHERE appName = 'My Web App' AND metricTimesliceName IN ('apm.service.transaction.duration') SINCE 1 hour ago"
    }
  }
}

# NRQL Queries for different monitoring scenarios

# 1. Performance Analysis Queries
"""
-- Top 10 slowest transactions
SELECT percentile(duration, 95), count(*)
FROM Transaction
WHERE appName = 'My Web App'
SINCE 1 hour ago
FACET name
LIMIT 10

-- Response time by environment
SELECT percentile(duration, 50, 90, 95)
FROM Transaction
WHERE appName = 'My Web App'
SINCE 1 day ago
FACET appName, environment

-- Error rate by transaction type
SELECT percentage(count(*), WHERE error IS true)
FROM Transaction
WHERE appName = 'My Web App'
SINCE 1 hour ago
FACET name
"""

# 2. Business Intelligence Queries
"""
-- Daily active users
SELECT uniqueCount(user_id)
FROM PageView
WHERE appName = 'My Web App'
SINCE 1 day ago

-- Conversion rate
SELECT count(*)/uniqueCount(session)
FROM Transaction, PageView
WHERE appName = 'My Web App'
AND name = 'WebTransaction/Custom/controller/checkout/complete'
SINCE 1 day ago

-- Revenue per user segment
SELECT sum(total_amount)
FROM OrderCreated
WHERE appName = 'My Web App'
SINCE 30 days ago
FACET user_segment
"""

# 3. Infrastructure Monitoring Queries
"""
-- CPU utilization by host
SELECT average(cpuPercent)
FROM SystemSample
WHERE appName = 'My Web App'
SINCE 1 hour ago
FACET hostname

-- Memory leak detection
SELECT average(memoryUsedPercent)
FROM SystemSample
WHERE appName = 'My Web App'
SINCE 24 hours ago
FACET hostname TIMESERIES 1 hour

-- Database performance
SELECT average(databaseDuration)
FROM Transaction
WHERE appName = 'My Web App'
SINCE 1 hour ago
FACET databaseName
"""

# 4. Custom Analytics Queries
"""
-- Feature usage
SELECT count(*)
FROM JavaScriptError, BrowserInteraction
WHERE appName = 'My Web App'
SINCE 24 hours ago
FACET eventType, targetName

-- Geographic distribution
SELECT count(*)
FROM PageView
WHERE appName = 'My Web App'
SINCE 24 hours ago
FACET countryCode

-- Device and browser breakdown
SELECT count(*)
FROM PageView
WHERE appName = 'My Web App'
SINCE 24 hours ago
FACET deviceType, userAgentName
"""

# Alert Conditions Configuration
resource "newrelic_nrql_alert_condition" "high_error_rate" {
  policy_id = newrelic_alert_policy.application_alerts.id
  name      = "High Error Rate"
  enabled   = true

  nrql {
    query = "SELECT percentage(count(*), WHERE error IS true) FROM Transaction WHERE appName = 'My Web App' AND metricTimesliceName IN ('apm.service.transaction.duration')"
  }

  critical {
    operator              = "above"
    threshold             = 5
    threshold_duration    = 300  # 5 minutes
    threshold_occurrences = "at_least_once"
  }

  warning {
    operator              = "above"
    threshold             = 2
    threshold_duration    = 600  # 10 minutes
    threshold_occurrences = "at_least_once"
  }
}

resource "newrelic_alert_policy" "application_alerts" {
  name = "Application Alert Policy"
  incident_preference = "PER_POLICY" # or "PER_CONDITION"
}

# Notification Channels
resource "newrelic_notification_destination" "slack_channel" {
  name = "Slack Alerts"
  type = "SLACK"

  property {
    key   = "url"
    value = "https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK"
  }
}

resource "newrelic_notification_channel" "slack_notifications" {
  name           = "Slack Notification Channel"
  type           = "SLACK"
  destination_id = newrelic_notification_destination.slack_channel.id

  product = "IINT" # Issues and Incidents
}

// New Relic Synthetic Monitor Scripts

// 1. Simple HTTP Status Check
var assert = require('assert');

// Configuration
var options = {
  // Monitor configuration
  name: "Homepage Status Check",
  type: "simple",
  frequency: 5, // minutes
  locations: ["AWS_EU_WEST_1", "AWS_US_EAST_1"],
  // HTTP request configuration
  url: "https://myapp.example.com",
  method: "GET",
  headers: {
    "User-Agent": "New Relic Synthetics"
  },
  // Response validation
  verifySSL: true,
  followRedirects: true,
  timeout: 10000 // 10 seconds
};

// Monitor execution
$http.get(options.url, options, function (err, response, body) {
  assert.ok(response.statusCode == 200,
           "Expected 200 OK response, got " + response.statusCode);

  assert.ok(body.length > 0, "Response body is empty");

  // Record custom metrics
  $util.insights.set("ResponseTime", response.elapsedTime);
  $util.insights.set("ContentLength", body.length);

  console.log("✅ Homepage is healthy");
});

// 2. API Endpoint Monitor with Validation
var assert = require('assert');
var JSON = require('json');

// Monitor configuration
var config = {
  name: "API Health Check",
  url: "https://api.myapp.example.com/health",
  headers: {
    "Authorization": "Bearer " + $secure.SYNTHETIC_API_TOKEN,
    "Content-Type": "application/json"
  },
  // Expected response structure
  expectedFields: ["status", "timestamp", "version"],
  expectedStatus: "healthy"
};

// Execute monitor
$http.get(config.url, { headers: config.headers }, function (err, response, body) {
  // Check response status
  assert.ok(response.statusCode == 200,
           "API returned status: " + response.statusCode);

  // Parse JSON response
  var data = JSON.parse(body);

  // Validate response structure
  config.expectedFields.forEach(function(field) {
    assert.ok(data.hasOwnProperty(field),
             "Missing required field: " + field);
  });

  // Validate health status
  assert.ok(data.status === config.expectedStatus,
           "Health status is: " + data.status + ", expected: " + config.expectedStatus);

  // Check if timestamp is recent (within 5 minutes)
  var now = new Date().getTime();
  var timestamp = new Date(data.timestamp).getTime();
  var timeDiff = Math.abs(now - timestamp);

  assert.ok(timeDiff < 300000, // 5 minutes
           "Health check timestamp is too old: " + new Date(data.timestamp));

  // Record custom metrics
  $util.insights.set("APIResponseTime", response.elapsedTime);
  $util.insights.set("APIVersion", data.version);
  $util.insights.set("HealthCheckAge", timeDiff);

  console.log("✅ API health check passed");
});

// 3. E-commerce Transaction Flow Monitor
var assert = require('assert');

// Test configuration
var testConfig = {
  baseUrl: "https://shop.example.com",
  userEmail: "[email protected]",
  userPassword: $secure.TEST_USER_PASSWORD,
  productIds: ["123", "456", "789"]
};

// Helper function to generate random data
function generateRandomEmail() {
  return "test_" + Date.now() + "@synthetic.com";
}

// Step 1: Browse products
$http.get(testConfig.baseUrl + "/products", function (err, response, body) {
  assert.ok(response.statusCode == 200, "Products page failed to load");

  // Parse product list
  var products = JSON.parse(body);
  assert.ok(products.length > 0, "No products found");

  // Select random product
  var product = products[Math.floor(Math.random() * products.length)];
  $util.insights.set("SelectedProduct", product.id);

  // Step 2: Add to cart
  $http.post(testConfig.baseUrl + "/cart/add", {
    json: {
      productId: product.id,
      quantity: 1
    }
  }, function (err, response, body) {
    assert.ok(response.statusCode == 200, "Failed to add product to cart");

    var cartResponse = JSON.parse(body);
    assert.ok(cartResponse.success, "Add to cart returned error");

    // Step 3: View cart
    $http.get(testConfig.baseUrl + "/cart", function (err, response, body) {
      assert.ok(response.statusCode == 200, "Failed to view cart");

      var cart = JSON.parse(body);
      assert.ok(cart.items.length > 0, "Cart is empty");

      // Step 4: Checkout
      $http.post(testConfig.baseUrl + "/checkout", {
        json: {
          email: generateRandomEmail(),
          firstName: "Synthetic",
          lastName: "Test",
          address: {
            street: "123 Test Street",
            city: "Test City",
            zipCode: "12345",
            country: "US"
          },
          paymentMethod: "test-card"
        }
      }, function (err, response, body) {
        assert.ok(response.statusCode == 200, "Checkout failed");

        var checkoutResponse = JSON.parse(body);
        assert.ok(checkoutResponse.success, "Checkout returned error");
        assert.ok(checkoutResponse.orderId, "No order ID returned");

        // Record success metrics
        $util.insights.set("CheckoutSuccess", true);
        $util.insights.set("OrderId", checkoutResponse.orderId);
        $util.insights.set("TotalTime", new Date().getTime() - testStartTime);

        console.log("✅ E-commerce flow completed successfully");
      });
    });
  });
});

var testStartTime = new Date().getTime();

// 4. Multi-step API Authentication Test
var assert = require('assert');
var crypto = require('crypto');

// Configuration
var authConfig = {
  apiUrl: "https://api.example.com",
  clientId: $secure.API_CLIENT_ID,
  clientSecret: $secure.API_CLIENT_SECRET
};

// Generate test data
var testUser = {
  email: "test_" + Date.now() + "@synthetic.com",
  password: crypto.randomBytes(16).toString('hex'),
  name: "Synthetic Test User"
};

// Step 1: Register user
$http.post(authConfig.apiUrl + "/auth/register", {
  json: {
    email: testUser.email,
    password: testUser.password,
    name: testUser.name
  }
}, function (err, response, body) {
  assert.ok(response.statusCode == 201, "User registration failed");

  var registerResponse = JSON.parse(body);
  assert.ok(registerResponse.success, "Registration returned error");

  // Step 2: Login
  $http.post(authConfig.apiUrl + "/auth/login", {
    json: {
      email: testUser.email,
      password: testUser.password
    }
  }, function (err, response, body) {
    assert.ok(response.statusCode == 200, "Login failed");

    var loginResponse = JSON.parse(body);
    assert.ok(loginResponse.token, "No access token returned");

    var authToken = loginResponse.token;

    // Step 3: Access protected resource
    $http.get(authConfig.apiUrl + "/user/profile", {
      headers: {
        "Authorization": "Bearer " + authToken
      }
    }, function (err, response, body) {
      assert.ok(response.statusCode == 200, "Protected resource access failed");

      var profile = JSON.parse(body);
      assert.ok(profile.email === testUser.email, "Profile data mismatch");

      // Step 4: Refresh token
      $http.post(authConfig.apiUrl + "/auth/refresh", {
        json: {
          token: authToken
        }
      }, function (err, response, body) {
        assert.ok(response.statusCode == 200, "Token refresh failed");

        var refreshResponse = JSON.parse(body);
        assert.ok(refreshResponse.token, "No new token returned");

        // Record metrics
        $util.insights.set("AuthFlowSuccess", true);
        $util.insights.set("TokenRefreshTime", response.elapsedTime);

        console.log("✅ Authentication flow completed successfully");
      });
    });
  });
});

// 5. Content and Performance Monitor
var assert = require('assert');

// Monitor configuration
var monitorConfig = {
  url: "https://blog.example.com/latest-article",
  performanceThresholds: {
    domContentLoaded: 2000, // ms
    loadComplete: 5000,      // ms
    firstPaint: 1000         // ms
  }
};

// Execute monitor with performance tracking
$webDriver.get(monitorConfig.url, function (err, driver) {
  assert.ok(!err, "Failed to load page: " + err);

  // Wait for page to load
  driver.waitForElementVisible("article", 10000, function (err) {
    assert.ok(!err, "Article element not found within 10 seconds");

    // Check content
    driver.getElementText("article h1", function (err, text) {
      assert.ok(!err, "Failed to get article title");
      assert.ok(text.length > 0, "Article title is empty");

      // Check for author information
      driver.elementIfExists(".author", function (err, element) {
        assert.ok(!err && element, "Author information not found");

        // Check for publication date
        driver.elementIfExists(".publish-date", function (err, element) {
          assert.ok(!err && element, "Publication date not found");

          // Get performance metrics
          driver.executeScript(function() {
            return {
              domContentLoaded: performance.timing.domContentLoadedEventEnd - performance.timing.navigationStart,
              loadComplete: performance.timing.loadEventEnd - performance.timing.navigationStart,
              firstPaint: performance.getEntriesByType('paint')[0]?.startTime || 0
            };
          }, function (err, metrics) {
            // Validate performance thresholds
            assert.ok(metrics.domContentLoaded < monitorConfig.performanceThresholds.domContentLoaded,
                     "DOM content load time exceeded threshold: " + metrics.domContentLoaded + "ms");

            assert.ok(metrics.loadComplete < monitorConfig.performanceThresholds.loadComplete,
                     "Page load time exceeded threshold: " + metrics.loadComplete + "ms");

            // Record performance metrics
            $util.insights.set("DOMLoadTime", metrics.domContentLoaded);
            $util.insights.set("PageLoadTime", metrics.loadComplete);
            $util.insights.set("FirstPaintTime", metrics.firstPaint);
            $util.insights.set("ArticleTitle", text);

            console.log("✅ Content and performance checks passed");
          });
        });
      });
    });
  });
});

// 6. Broken Link Checker
var assert = require('assert');

var linkCheckerConfig = {
  baseUrl: "https://myapp.example.com",
  maxRedirects: 5,
  timeout: 10000,
  // Links to check
  links: [
    "/",
    "/about",
    "/contact",
    "/products",
    "/blog",
    "/pricing",
    "/faq"
  ]
};

// Check each link
var results = [];
var completedChecks = 0;

linkCheckerConfig.links.forEach(function(link, index) {
  var fullUrl = linkCheckerConfig.baseUrl + link;

  $http.get(fullUrl, {
    timeout: linkCheckerConfig.timeout,
    maxRedirects: linkCheckerConfig.maxRedirects
  }, function (err, response, body) {
    completedChecks++;

    results.push({
      url: fullUrl,
      statusCode: response ? response.statusCode : 0,
      responseTime: response ? response.elapsedTime : 0,
      error: err ? err.message : null
    });

    // If all checks complete, analyze results
    if (completedChecks === linkCheckerConfig.links.length) {
      analyzeResults();
    }
  });
});

function analyzeResults() {
  var successCount = 0;
  var totalTime = 0;
  var failedLinks = [];

  results.forEach(function(result) {
    if (result.statusCode >= 200 && result.statusCode < 400) {
      successCount++;
      totalTime += result.responseTime;
    } else {
      failedLinks.push({
        url: result.url,
        statusCode: result.statusCode,
        error: result.error
      });
    }
  });

  // Assertions
  assert.ok(successCount === results.length,
           "Some links failed: " + JSON.stringify(failedLinks));

  // Record metrics
  $util.insights.set("LinkCheckSuccess", successCount === results.length);
  $util.insights.set("CheckedLinks", results.length);
  $util.insights.set("AverageResponseTime", totalTime / results.length);
  $util.insights.set("FailedLinks", failedLinks.length);

  console.log("✅ All links checked successfully");
}