run basic pipeline.yml

.github/workflows/pipeline.yml

@@ -1,141 +1,152 @@
-name: CI/CD Pipeline with Monitoring
+name: Enhanced Java Application Pipeline with Metrics
 
 on:
   push:
-    branches: [ pipeline-optimization  ]
+    branches: [ main ]
   pull_request:
-    branches: [ pipeline-optimization ]
-
-env:
-  PROMETHEUS_URL: http://localhost:9090
-  GRAFANA_URL: http://localhost:3000
+    branches: [ main ]
 
 jobs:
-  build:
+  build-with-metrics:
     runs-on: ubuntu-latest
+
     services:
-      mysql:
-        image: mysql:9.1
-        env:
-          MYSQL_ROOT_PASSWORD: ''
-          MYSQL_ALLOW_EMPTY_PASSWORD: true
-          MYSQL_USER: petclinic
-          MYSQL_PASSWORD: petclinic
-          MYSQL_DATABASE: petclinic
+      prometheus:
+        image: prom/prometheus:latest
         ports:
-          - 3306:3306
+          - 9090:9090
+        healthcheck:
+          test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/healthy"]
+          interval: 10s
+          timeout: 5s
+          retries: 3
+
+      pushgateway:
+        image: prom/pushgateway:latest
+        ports:
+          - 9091:9091
+        healthcheck:
+          test: ["CMD", "wget", "-q", "--spider", "http://localhost:9091/-/healthy"]
+          interval: 10s
+          timeout: 5s
+          retries: 3
 
     steps:
-      - name: Start Monitoring Stack
-        run: |
-          docker-compose up -d prometheus grafana powerapi node-exporter
-          sleep 10 # Attendre que les services démarrent
+      - uses: actions/checkout@v4
 
-      - name: Checkout code
-        uses: actions/checkout@v4
+      # Installation and setup of monitoring tools
+      - name: Setup monitoring tools
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y powerstat linux-tools-common linux-tools-generic
+          sudo snap install powerapi
+          curl -L https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz -o node_exporter.tar.gz
+          tar xvfz node_exporter.tar.gz
+
+      # Start monitoring tools with improved configuration
+      - name: Start monitoring
+        run: |
+          # Start PowerAPI with Prometheus output
+          sudo powerapi --pid $$ --frequency 1000 --output prometheus --pushgateway-url http://localhost:9091/metrics/job/powerapi &
+          echo "POWERAPI_PID=$!" >> $GITHUB_ENV
+          
+          # Start node exporter
+          ./node_exporter-*/node_exporter --web.listen-address=":9100" &
+          echo "NODE_EXPORTER_PID=$!" >> $GITHUB_ENV
+          
+          # Create start timestamp file
+          date +%s%N > pipeline_start_time.txt
 
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
           java-version: '17'
           distribution: 'adopt'
+          cache: maven
 
-      - name: Record Build Start
-        run: |
-          curl -X POST http://localhost:9091/metrics/job/pipeline/instance/build \
-          --data-binary "pipeline_stage_start{stage=\"build\"} $(date +%s)"
-
-      - name: Maven Build
+      - name: Build with Maven
         run: |
           start_time=$(date +%s%N)
-          ./mvnw clean package
+          ./mvnw -B verify
           end_time=$(date +%s%N)
-          duration=$(( ($end_time - $start_time)/1000000 ))
-          
-          # Envoyer les métriques à Prometheus
-          echo "pipeline_build_duration_ms $duration" | curl -X POST --data-binary @- \
-            http://localhost:9091/metrics/job/pipeline/instance/build
+          echo "BUILD_TIME=$((($end_time - $start_time)/1000000))" >> $GITHUB_ENV
 
-      - name: Run Tests
+      - name: Run tests
         run: |
           start_time=$(date +%s%N)
           ./mvnw test
           end_time=$(date +%s%N)
-          duration=$(( ($end_time - $start_time)/1000000 ))
-          
-          # Envoyer les métriques à Prometheus
-          echo "pipeline_test_duration_ms $duration" | curl -X POST --data-binary @- \
-            http://localhost:9091/metrics/job/pipeline/instance/test
+          echo "TEST_TIME=$((($end_time - $start_time)/1000000))" >> $GITHUB_ENV
 
-      - name: Collect Metrics
+      - name: Build Docker image
+        run: |
+          start_time=$(date +%s%N)
+          docker build -t app:latest .
+          end_time=$(date +%s%N)
+          echo "DOCKER_BUILD_TIME=$((($end_time - $start_time)/1000000))" >> $GITHUB_ENV
+
+      - name: Setup Kubernetes
+        uses: helm/kind-action@v1
+
+      - name: Deploy to Kubernetes
+        run: |
+          start_time=$(date +%s%N)
+          kubectl apply -f k8s/
+          kubectl wait --for=condition=ready pod -l app=petclinic --timeout=180s
+          end_time=$(date +%s%N)
+          echo "DEPLOY_TIME=$((($end_time - $start_time)/1000000))" >> $GITHUB_ENV
+
+      # Export metrics with improved labeling and job naming
+      - name: Export metrics to Prometheus
+        run: |
+          # Export timing metrics with descriptive labels
+          echo "pipeline_build_duration_ms{stage=\"build\",project=\"petclinic\"} ${{ env.BUILD_TIME }}" | curl --data-binary @- http://localhost:9091/metrics/job/petclinic-pipeline
+          echo "pipeline_test_duration_ms{stage=\"test\",project=\"petclinic\"} ${{ env.TEST_TIME }}" | curl --data-binary @- http://localhost:9091/metrics/job/petclinic-pipeline
+          echo "pipeline_docker_build_duration_ms{stage=\"docker-build\",project=\"petclinic\"} ${{ env.DOCKER_BUILD_TIME }}" | curl --data-binary @- http://localhost:9091/metrics/job/petclinic-pipeline
+          echo "pipeline_deploy_duration_ms{stage=\"deploy\",project=\"petclinic\"} ${{ env.DEPLOY_TIME }}" | curl --data-binary @- http://localhost:9091/metrics/job/petclinic-pipeline
+          
+          # Export power consumption metrics
+          while IFS=, read -r timestamp watts; do
+            echo "power_consumption_watts{project=\"petclinic\"} $watts $timestamp" | curl --data-binary @- http://localhost:9091/metrics/job/petclinic-pipeline
+          done < energy_metrics.csv
+
+      # Collect additional resource metrics
+      - name: Collect resource metrics
+        run: |
+          # Memory usage metric
+          echo "pipeline_memory_usage_bytes{project=\"petclinic\"} $(free -b | grep Mem: | awk '{print $3}')" | curl --data-binary @- http://localhost:9091/metrics/job/petclinic-pipeline
+          
+          # CPU usage metric
+          echo "pipeline_cpu_usage_percent{project=\"petclinic\"} $(top -bn1 | grep "Cpu(s)" | awk '{print $2}')" | curl --data-binary @- http://localhost:9091/metrics/job/petclinic-pipeline
+
+      # Stop monitoring tools and collect metrics
+      - name: Collect metrics
         if: always()
         run: |
-          # Récupérer les métriques de PowerAPI
-          curl -o power_metrics.json http://localhost:9091/metrics/job/powerapi
+          # End timestamp
+          date +%s%N > pipeline_end_time.txt
           
-          # Récupérer les métriques système
-          curl -o node_metrics.json http://localhost:9100/metrics
+          # Stop PowerAPI
+          sudo kill ${{ env.POWERAPI_PID }}
           
-          # Récupérer les métriques de Prometheus
-          curl -o prometheus_metrics.json "$PROMETHEUS_URL/api/v1/query?query=pipeline_build_duration_ms"
+          # Stop node exporter
+          kill ${{ env.NODE_EXPORTER_PID }}
           
-          # Générer le rapport des métriques
-          cat << EOF > metrics_report.txt
-          Pipeline Run Report - $(date)
-          ===========================
-          
-          Build Duration: $(cat prometheus_metrics.json | jq .data.result[0].value[1])ms
-          Power Consumption: $(cat power_metrics.json | jq .total_power_consumption)W
-          CPU Usage: $(cat node_metrics.json | grep cpu_usage_percent | awk '{print $2}')%
-          Memory Usage: $(cat node_metrics.json | grep memory_usage_bytes | awk '{print $2/1024/1024}')MB
-          EOF
+          # Collect system metrics
+          top -b -n 1 > system_metrics.txt
+          free -m > memory_metrics.txt
+          df -h > disk_metrics.txt
 
-      - name: Upload Metrics
+      # Save metrics as artifacts
+      - name: Save metrics
+        if: always()
         uses: actions/upload-artifact@v3
         with:
           name: pipeline-metrics
           path: |
-            metrics_report.txt
-            power_metrics.json
-            node_metrics.json
-            prometheus_metrics.json
-
-      - name: Create Grafana Dashboard
-        if: always()
-        run: |
-          # Créer un dashboard Grafana via l'API
-          curl -X POST -H "Content-Type: application/json" \
-            -d @- \
-            $GRAFANA_URL/api/dashboards/db << EOF
-          {
-            "dashboard": {
-              "title": "Pipeline Performance Dashboard",
-              "panels": [
-                {
-                  "title": "Build Duration",
-                  "type": "graph",
-                  "datasource": "Prometheus",
-                  "targets": [
-                    {
-                      "expr": "pipeline_build_duration_ms"
-                    }
-                  ]
-                },
-                {
-                  "title": "Power Consumption",
-                  "type": "graph",
-                  "datasource": "Prometheus",
-                  "targets": [
-                    {
-                      "expr": "power_consumption_watts"
-                    }
-                  ]
-                }
-              ]
-            }
-          }
-          EOF
-
-      - name: Cleanup
-        if: always()
-        run: docker-compose down
+            energy_metrics.csv
+            system_metrics.txt
+            memory_metrics.txt
+            disk_metrics.txt
+            pipeline_start_time.txt
+            pipeline_end_time.txt

docker-compose.yml

@@ -1,6 +1,7 @@
 version: '3.8'
 
 services:
+  # Database
   mysql:
     image: mysql:9.1
     ports:
@@ -13,44 +14,54 @@ services:
       - MYSQL_DATABASE=petclinic
     volumes:
       - "./conf.d:/etc/mysql/conf.d:ro"
-
   postgres:
-    image: postgres:17.0
-    ports:
-      - "5432:5432"
-    environment:
-      - POSTGRES_PASSWORD=petclinic
-      - POSTGRES_USER=petclinic
-      - POSTGRES_DB=petclinic
+      image: postgres:17.0
+      ports:
+        - "5432:5432"
+      environment:
+        - POSTGRES_PASSWORD=petclinic
+        - POSTGRES_USER=petclinic
+        - POSTGRES_DB=petclinic
 
-  # Ajout de Prometheus pour la collecte des métriques
   prometheus:
     image: prom/prometheus:latest
     ports:
       - "9090:9090"
     volumes:
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./monitoring/prometheus/pipeline-rules.yml:/etc/prometheus/pipeline-rules.yml:ro
       - prometheus_data:/prometheus
     command:
       - '--config.file=/etc/prometheus/prometheus.yml'
       - '--storage.tsdb.path=/prometheus'
-      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
-      - '--web.console.templates=/usr/share/prometheus/consoles'
+      - '--web.enable-lifecycle'
+      - '--log.level=debug'
+    user: root  # Added this line
+    healthcheck:
+      test: [ "CMD", "wget", "-q", "--spider", "http://localhost:9090/-/healthy" ]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    networks:
+      - monitoring-network
 
-  # Ajout de Grafana pour la visualisation
   grafana:
     image: grafana/grafana:latest
     ports:
       - "3000:3000"
     volumes:
+      - ./monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards
       - grafana_data:/var/lib/grafana
     environment:
       - GF_SECURITY_ADMIN_PASSWORD=admin
       - GF_USERS_ALLOW_SIGN_UP=false
+      - GF_INSTALL_PLUGINS=grafana-piechart-panel
     depends_on:
-      - prometheus
+      prometheus:
+        condition: service_healthy
+    networks:
+      - monitoring-network
 
-  # PowerAPI pour la mesure de consommation énergétique
   powerapi:
     image: powerapi/powerapi:latest
     privileged: true
@@ -61,11 +72,19 @@ services:
     environment:
       - POWERAPI_CPU_EVENTS=cpu-cycles,instructions
       - POWERAPI_PUSH_FREQUENCY=1000
-      - POWERAPI_PUSH_URL=http://prometheus:9091/metrics/job/powerapi
+      - POWERAPI_PUSH_URL=http://pushgateway:9091/metrics/job/powerapi
     depends_on:
-      - prometheus
+      - pushgateway
+    networks:
+      - monitoring-network
+
+  pushgateway:
+    image: prom/pushgateway
+    ports:
+      - "9091:9091"
+    networks:
+      - monitoring-network
 
-  # Node exporter pour les métriques système
   node-exporter:
     image: prom/node-exporter:latest
     volumes:
@@ -78,7 +97,15 @@ services:
       - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
     ports:
       - "9100:9100"
+    depends_on:
+      - prometheus
+    networks:
+      - monitoring-network
 
 volumes:
   prometheus_data:
   grafana_data:
+
+networks:
+  monitoring-network:
+    driver: bridge

monitoring/grafana/dashboards/pipeline.json

@@ -0,0 +1,213 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max", "min"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "pipeline_build_duration_ms",
+          "legendFormat": "Build Duration",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Pipeline Build Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "smooth",
+            "lineWidth": 2,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "watts"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": ["mean", "max"],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "power_consumption_watts",
+          "legendFormat": "Power Consumption",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Power Consumption",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": ["pipeline", "performance"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-1h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Pipeline Performance",
+  "weekStart": ""
+}

monitoring/prometheus/pipeline-rules.yml

@@ -0,0 +1,29 @@
+groups:
+  - name: pipeline_performance
+    rules:
+      # Durée moyenne du build
+      - record: pipeline:build_duration:avg_5m
+        expr: avg_over_time(pipeline_build_duration_ms[5m])
+
+      # Consommation énergétique totale
+      - record: pipeline:power_consumption:total
+        expr: sum(power_consumption_watts{job="powerapi"})
+
+      # Alertes
+      - alert: HighBuildTime
+        expr: pipeline_build_duration_ms > 300000
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Build duration too high"
+          description: "Build is taking more than 5 minutes"
+
+      - alert: HighPowerConsumption
+        expr: power_consumption_watts > 100
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High power consumption detected"
+          description: "Power consumption is above 100W for more than 2 minutes"

monitoring/prometheus/prometheus.yml

@@ -0,0 +1,38 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+rule_files:
+  - "pipeline-rules.yml"
+
+scrape_configs:
+  # Pipeline metrics
+  - job_name: 'petclinic-pipeline'
+    static_configs:
+      - targets: ['pushgateway:9091']
+        labels:
+          application: 'petclinic'
+          type: 'pipeline'
+
+  # PowerAPI metrics
+  - job_name: 'powerapi'
+    static_configs:
+      - targets: ['powerapi:9091']
+    metrics_path: '/metrics'
+
+  # Node Exporter metrics
+  - job_name: 'node'
+    static_configs:
+      - targets: ['node-exporter:9100']
+
+  # Application metrics
+  - job_name: 'spring-petclinic'
+    metrics_path: '/actuator/prometheus'
+    static_configs:
+      - targets: ['host.docker.internal:8080']
+
+  # Database metrics
+  - job_name: 'mysql'
+    metrics_path: '/metrics'
+    static_configs:
+      - targets: ['mysql:3306']