diff --git a/ProcédureGit.md b/ProcédureGit.md new file mode 100644 index 0000000..7f1a789 --- /dev/null +++ b/ProcédureGit.md @@ -0,0 +1,32 @@ +# Test rapide avant d'enregistrer : + + - Redémarrer le docker principale + cd C:\DevOpsProject\docker\ + docker-compose -f docker-compose.yml down --remove-orphans + docker-compose -f docker-compose.yml up -d + + - Redémarrer les autres dockers + cd C:\DevOpsProject\scripts\ + .\start_other_ctnrs.ps1 + +# Pour afficher tous les conteneurs Docker, ainsi que les nœuds et pods Kubernetes : + - Conteneurs Docker + docker ps -a + + - Nœuds Kubernetes + kubectl get nodes + + - Pods Kubernetes + kubectl get pods --all-namespaces + +✅ Docker : Tous les services (ELK, Gitea, NGINX, base de données, monitoring) sont Up et fonctionnels. +✅ Kubernetes : Les nœuds sont Ready, les pods système sont Running. + +# Pusher le projet modifié dans git : + - Aller dans le dossier du projet + - cliq droit "git bash here" + - "git add ." + - "git commit -m "détail des modifs" + - "git push origin main" + + diff --git a/docker/docker-compose-db.yml b/docker/docker-compose-db.yml index a2159dd..9e76bd4 100644 --- a/docker/docker-compose-db.yml +++ b/docker/docker-compose-db.yml @@ -1,8 +1,8 @@ x-logging: &default-logging - driver: fluentd + driver: json-file options: - fluentd-address: localhost:24224 - tag: devops.* + max-size: "10m" + max-file: "5" services: db: diff --git a/docker/docker-compose-elk.yml b/docker/docker-compose-elk.yml index 7fdf937..c1ff276 100644 --- a/docker/docker-compose-elk.yml +++ b/docker/docker-compose-elk.yml @@ -13,6 +13,12 @@ services: hard: -1 volumes: - elasticsearch-data:/usr/share/elasticsearch/data + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:9200/_cluster/health?wait_for_status=yellow || exit 1"] + interval: 10s + timeout: 10s + retries: 10 + start_period: 30s ports: - "9200:9200" networks: @@ -24,6 +30,8 @@ services: environment: - SERVER_HOST=0.0.0.0 - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + volumes: + - ../monitoring/elklogs/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro ports: - "5601:5601" networks: @@ -37,14 +45,16 @@ services: environment: - LS_JAVA_OPTS=-Xms512m -Xmx512m volumes: - - /mnt/c/DevOpsProject/monitoring/elklogs/logstash/config:/usr/share/logstash/pipeline:ro - - /mnt/c/DevOpsProject/monitoring/elklogs/logstash/config/jvm.options:/usr/share/logstash/config/jvm.options:ro + - ../monitoring/elklogs/logstash/pipeline/pipelines.yml:/usr/share/logstash/pipeline/pipelines.yml:ro + - ../monitoring/elklogs/logstash/config/pipelines.yml:/usr/share/logstash/config/pipelines.yml:ro + - ../monitoring/elklogs/logstash/config/jvm.options:/usr/share/logstash/config/jvm.options:ro ports: - "5044:5044" networks: - elk depends_on: - - elasticsearch + elasticsearch: + condition: service_healthy filebeat: image: docker.elastic.co/beats/filebeat:8.17.0 diff --git a/docker/docker-compose-gitea.yml b/docker/docker-compose-gitea.yml index 7696b6a..7997c41 100644 --- a/docker/docker-compose-gitea.yml +++ b/docker/docker-compose-gitea.yml @@ -1,8 +1,8 @@ x-logging: &default-logging - driver: fluentd + driver: json-file options: - fluentd-address: localhost:24224 - tag: devops.* + max-size: "10m" + max-file: "5" services: gitea: diff --git a/docker/docker-compose-nginx.yml b/docker/docker-compose-nginx.yml index 7cfd89b..0b612c5 100644 --- a/docker/docker-compose-nginx.yml +++ b/docker/docker-compose-nginx.yml @@ -1,8 +1,8 @@ x-logging: &default-logging - driver: fluentd + driver: json-file options: - fluentd-address: localhost:24224 - tag: devops.* + max-size: "10m" + max-file: "5" services: nginx: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c583741..e4704b4 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,8 +1,8 @@ x-logging: &default-logging - driver: fluentd + driver: json-file options: - fluentd-address: localhost:24224 - tag: devops.* + max-size: "10m" + max-file: "5" services: app: diff --git a/gitea/gitea-data/gitea/gitea.db b/gitea/gitea-data/gitea/gitea.db index 7f5406c..1b24f57 100644 Binary files a/gitea/gitea-data/gitea/gitea.db and b/gitea/gitea-data/gitea/gitea.db differ diff --git a/gitea/gitea-data/gitea/indexers/issues.bleve/store/root.bolt b/gitea/gitea-data/gitea/indexers/issues.bleve/store/root.bolt index 10d08ae..a706c2d 100644 Binary files a/gitea/gitea-data/gitea/indexers/issues.bleve/store/root.bolt and b/gitea/gitea-data/gitea/indexers/issues.bleve/store/root.bolt differ diff --git a/gitea/gitea-data/gitea/queues/common/000023.log b/gitea/gitea-data/gitea/queues/common/000047.log similarity index 100% rename from gitea/gitea-data/gitea/queues/common/000023.log rename to gitea/gitea-data/gitea/queues/common/000047.log diff --git a/gitea/gitea-data/gitea/queues/common/CURRENT b/gitea/gitea-data/gitea/queues/common/CURRENT index 1e21b78..a9ebced 100644 --- a/gitea/gitea-data/gitea/queues/common/CURRENT +++ b/gitea/gitea-data/gitea/queues/common/CURRENT @@ -1 +1 @@ -MANIFEST-000024 +MANIFEST-000048 diff --git a/gitea/gitea-data/gitea/queues/common/CURRENT.bak b/gitea/gitea-data/gitea/queues/common/CURRENT.bak index 5af92b2..a6d5883 100644 --- a/gitea/gitea-data/gitea/queues/common/CURRENT.bak +++ b/gitea/gitea-data/gitea/queues/common/CURRENT.bak @@ -1 +1 @@ -MANIFEST-000022 +MANIFEST-000046 diff --git a/gitea/gitea-data/gitea/queues/common/LOG b/gitea/gitea-data/gitea/queues/common/LOG index 73ae5b6..cba671e 100644 --- a/gitea/gitea-data/gitea/queues/common/LOG +++ b/gitea/gitea-data/gitea/queues/common/LOG @@ -100,3 +100,111 @@ 07:31:18.922596 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] 07:31:18.962518 db@janitor F·3 G·0 07:31:18.962774 db@open done T·47.494468ms +=============== Jan 30, 2026 (UTC) =============== +13:54:27.868593 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +13:54:27.892455 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +13:54:27.892849 db@open opening +13:54:27.894307 journal@recovery F·1 +13:54:27.894613 journal@recovery recovering @23 +13:54:27.897603 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +13:54:27.947761 db@janitor F·3 G·0 +13:54:27.947926 db@open done T·54.793348ms +=============== Jan 30, 2026 (UTC) =============== +14:15:33.586338 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +14:15:33.592592 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +14:15:33.598256 db@open opening +14:15:33.599579 journal@recovery F·1 +14:15:33.599862 journal@recovery recovering @25 +14:15:33.602391 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +14:15:33.640088 db@janitor F·3 G·0 +14:15:33.640289 db@open done T·41.7863ms +=============== Jan 30, 2026 (UTC) =============== +14:17:09.197281 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +14:17:09.202591 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +14:17:09.202962 db@open opening +14:17:09.204108 journal@recovery F·1 +14:17:09.204373 journal@recovery recovering @27 +14:17:09.207398 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +14:17:09.239988 db@janitor F·3 G·0 +14:17:09.240147 db@open done T·37.04192ms +=============== Jan 30, 2026 (UTC) =============== +14:27:40.555685 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +14:27:40.563000 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +14:27:40.563425 db@open opening +14:27:40.564901 journal@recovery F·1 +14:27:40.565188 journal@recovery recovering @29 +14:27:40.568025 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +14:27:40.618333 db@janitor F·3 G·0 +14:27:40.618563 db@open done T·54.973788ms +=============== Jan 30, 2026 (UTC) =============== +14:33:00.048029 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +14:33:00.070767 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +14:33:00.072809 db@open opening +14:33:00.074262 journal@recovery F·1 +14:33:00.075293 journal@recovery recovering @31 +14:33:00.095031 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +14:33:00.177600 db@janitor F·3 G·0 +14:33:00.178954 db@open done T·105.87586ms +=============== Jan 30, 2026 (UTC) =============== +14:37:56.105146 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +14:37:56.143515 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +14:37:56.144085 db@open opening +14:37:56.158985 journal@recovery F·1 +14:37:56.159486 journal@recovery recovering @33 +14:37:56.168153 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +14:37:56.245259 db@janitor F·3 G·0 +14:37:56.245604 db@open done T·101.341687ms +=============== Jan 30, 2026 (UTC) =============== +14:58:51.261319 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +14:58:51.268236 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +14:58:51.268769 db@open opening +14:58:51.270343 journal@recovery F·1 +14:58:51.270594 journal@recovery recovering @35 +14:58:51.273508 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +14:58:51.304932 db@janitor F·3 G·0 +14:58:51.305182 db@open done T·36.20052ms +=============== Jan 30, 2026 (UTC) =============== +15:02:02.430077 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:02:02.436038 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +15:02:02.436457 db@open opening +15:02:02.438119 journal@recovery F·1 +15:02:02.438396 journal@recovery recovering @37 +15:02:02.440960 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +15:02:02.471433 db@janitor F·3 G·0 +15:02:02.471668 db@open done T·35.052337ms +=============== Jan 30, 2026 (UTC) =============== +15:03:56.374460 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:03:56.380375 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +15:03:56.380849 db@open opening +15:03:56.382082 journal@recovery F·1 +15:03:56.382416 journal@recovery recovering @39 +15:03:56.385389 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +15:03:56.418633 db@janitor F·3 G·0 +15:03:56.418842 db@open done T·37.830034ms +=============== Jan 30, 2026 (UTC) =============== +15:05:39.076957 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:05:39.083558 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +15:05:39.084101 db@open opening +15:05:39.086277 journal@recovery F·1 +15:05:39.086634 journal@recovery recovering @41 +15:05:39.090198 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +15:05:39.126223 db@janitor F·3 G·0 +15:05:39.126403 db@open done T·42.051216ms +=============== Jan 30, 2026 (UTC) =============== +15:12:38.550365 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +15:12:38.572801 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +15:12:38.573301 db@open opening +15:12:38.575371 journal@recovery F·1 +15:12:38.576023 journal@recovery recovering @43 +15:12:38.580508 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +15:12:38.617431 db@janitor F·3 G·0 +15:12:38.621316 db@open done T·47.850414ms +=============== Jan 30, 2026 (UTC) =============== +16:51:56.898330 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +16:51:56.914950 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +16:51:56.915351 db@open opening +16:51:56.919693 journal@recovery F·1 +16:51:56.919967 journal@recovery recovering @45 +16:51:56.927562 version@stat F·[0 1] S·577B[0B 577B] Sc·[0.00 0.00] +16:51:56.988766 db@janitor F·3 G·0 +16:51:56.988996 db@open done T·73.514866ms diff --git a/gitea/gitea-data/gitea/queues/common/MANIFEST-000024 b/gitea/gitea-data/gitea/queues/common/MANIFEST-000048 similarity index 59% rename from gitea/gitea-data/gitea/queues/common/MANIFEST-000024 rename to gitea/gitea-data/gitea/queues/common/MANIFEST-000048 index b2c184d..5802584 100644 Binary files a/gitea/gitea-data/gitea/queues/common/MANIFEST-000024 and b/gitea/gitea-data/gitea/queues/common/MANIFEST-000048 differ diff --git a/gitea/gitea-data/gitea/sessions/3/f/3fb760b8668882c2 b/gitea/gitea-data/gitea/sessions/3/f/3fb760b8668882c2 deleted file mode 100644 index 9433983..0000000 Binary files a/gitea/gitea-data/gitea/sessions/3/f/3fb760b8668882c2 and /dev/null differ diff --git a/monitoring/elklogs/kibana/config/kibana.yml b/monitoring/elklogs/kibana/config/kibana.yml new file mode 100644 index 0000000..c5eec7e --- /dev/null +++ b/monitoring/elklogs/kibana/config/kibana.yml @@ -0,0 +1,31 @@ +# =================================== Kibana Configuration =================================== +# Configuration principale pour Kibana 8.17 + +# --- Serveur --- +server.host: "0.0.0.0" # Écoute sur toutes les interfaces réseau +server.port: 5601 # Port d'accès à l'interface web +server.name: "kibana-devops" # Nom affiché dans les logs et l'UI + +# --- Elasticsearch --- +elasticsearch.hosts: ["http://elasticsearch:9200"] # URL du cluster Elasticsearch +elasticsearch.requestTimeout: 30000 # Délai max pour les requêtes (ms) + +# --- Sécurité --- +xpack.encryptedSavedObjects.encryptionKey: "b524234a31e5bf211a0bb157192b6faf" +# Chiffre les objets sauvegardés (dashboards, visualisations) + +xpack.reporting.encryptionKey: "111d1cb71e564e0ac3aafae85151b895" +# Chiffre les rapports générés par Reporting + +xpack.security.encryptionKey: "ecb7071b9737158642ca555ce25206fd" +# Chiffre les sessions utilisateur + +# --- Fonctionnalités avancées --- +xpack.reporting.roles.enabled: false # ✅ Désactive le système de rôles déprécié +# Active le nouveau système basé sur les privilèges Kibana (requis pour les futures versions) + +monitoring.kibana.collection.enabled: true # Active la collecte des métriques Kibana +# Affiche les performances de Kibana dans l'onglet Monitoring + +# --- Interface --- +i18n.locale: "fr-FR" # Langue par défaut de l'interface \ No newline at end of file diff --git a/monitoring/elklogs/logstash/config/jvm.options b/monitoring/elklogs/logstash/config/jvm.options index 9c56234..5233ca1 100644 --- a/monitoring/elklogs/logstash/config/jvm.options +++ b/monitoring/elklogs/logstash/config/jvm.options @@ -1,6 +1,8 @@ --Xms1g --Xmx1g --Djava.awt.headless=true --Dfile.encoding=UTF-8 +-Xms512m +-Xmx512m +-XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError --Djava.security.egd=file:/dev/urandom +-XX:HeapDumpPath=/usr/share/logstash/logs/heap_dump.hprof +-XX:MaxMetaspaceSize=256m +-XX:+DisableExplicitGC +-XX:+AlwaysPreTouch \ No newline at end of file diff --git a/monitoring/elklogs/logstash/config/pipelines.yml b/monitoring/elklogs/logstash/config/pipelines.yml new file mode 100644 index 0000000..2d5e5b5 --- /dev/null +++ b/monitoring/elklogs/logstash/config/pipelines.yml @@ -0,0 +1,14 @@ +# pipelines.yml — Fichier de configuration principale pour Logstash +# Il définit quels pipelines (flux de traitement) Logstash doit charger. +# Ce fichier ne remplace pas logstash.conf, il le référence. + +- pipeline.id: main + # ✅ Identifiant unique obligatoire du pipeline + path.config: "/usr/share/logstash/pipeline/logstash.conf" + # ✅ Chemin vers le fichier .conf à charger (doit exister dans le conteneur) + pipeline.workers: 2 + # Nombre de threads pour traiter les événements (par défaut: nombre de CPU) + pipeline.batch.size: 125 + # Taille du lot d'événements traités ensemble + queue.type: memory + # Type de file d'attente : "memory" (par défaut) ou "persisted" (sur disque) \ No newline at end of file diff --git a/monitoring/elklogs/logstash/pipeline/logstash.conf b/monitoring/elklogs/logstash/pipeline/logstash.conf new file mode 100644 index 0000000..02a3389 --- /dev/null +++ b/monitoring/elklogs/logstash/pipeline/logstash.conf @@ -0,0 +1,92 @@ +# =================================== +# Fichier : logstash.conf +# Description : Configuration complète de Logstash +# Auteur : DevOps Project +# Date : 30 janvier 2026 +# =================================== + +# ——— SECTION INPUT ——— +# Définit les sources des logs +input { + # Entrée via Beats (Filebeat, Metricbeat, etc.) – recommandé + beats { + port => 5044 + # host => "0.0.0.0" # Écoute sur toutes les interfaces + } + + # Exemple : lecture de fichiers logs locaux + # file { + # path => ["/var/log/*.log", "/app/logs/*.log"] + # start_position => "beginning" + # sincedb_path => "/dev/null" # Pour relire les fichiers à chaque démarrage (développement) + # } + + # Exemple : réception de logs Syslog via UDP/TCP + # syslog { + # port => 514 + # type => "syslog" + # } +} + +# ——— SECTION FILTER ——— +# Traitement, parsing et enrichissement des logs +filter { + # Exemple : filtrer uniquement les logs de type "nginx" + # if [type] == "nginx" { + # grok { + # match => { "message" => "%{COMBINEDAPACHELOG}" } + # } + # date { + # match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ] + # } + # } + + # Exemple : parser des logs JSON + # json { + # source => "message" + # } + + # Ajout de champs métadonnées + mutate { + add_field => { + "environment" => "production" + "project" => "DevOpsProject" + } + # Supprimer des champs inutiles + # remove_field => ["agent", "input_type"] + } + + # Gestion des erreurs de parsing + # if "_grokparsefailure" in [tags] { + # drop {} + # } +} + +# ——— SECTION OUTPUT ——— +# Envoi des logs vers une destination +output { + # Sortie vers Elasticsearch + elasticsearch { + hosts => ["http://elasticsearch:9200"] + index => "logs-%{+YYYY.MM.dd}" + # user => "elastic" + # password => "changeme" + } + + # Sortie vers la console (utile pour le debug) + stdout { + codec => rubydebug + } + + # Exemple : sortie vers un fichier + # file { + # path => "/logs/parsed-%{+YYYY-MM-dd}.log" + # codec => json + # } +} + +# ——— CONSEILS ——— +# - Placez ce fichier dans : ./pipeline/logstash.conf +# - Montez le dossier dans le conteneur Docker +# - Assurez-vous qu’Elasticsearch est prêt avant Logstash +# - Utilisez `depends_on` avec condition de santé dans docker-compose.yml \ No newline at end of file diff --git a/scripts/start_other_ctnrs.ps1 b/scripts/start_other_ctnrs.ps1 index 44db06a..1034f9c 100644 --- a/scripts/start_other_ctnrs.ps1 +++ b/scripts/start_other_ctnrs.ps1 @@ -1,7 +1,4 @@ -# Définit le chemin vers le dossier contenant tous tes fichiers docker-compose.yml $projectPath = "C:\DevOpsProject\docker" - -# La liste de tous tes fichiers docker-compose YAML à lancer $composeFiles = @( "docker-compose-db.yml", "docker-compose-elk.yml", @@ -9,12 +6,17 @@ $composeFiles = @( "docker-compose-nginx.yml" ) -# Pour chaque fichier dans la liste foreach ($file in $composeFiles) { - # Lance une nouvelle instance de 'docker-compose' pour ce fichier, en mode détaché (-d) - # Start-Process permet de lancer la commande en arrière-plan, sans bloquer le script - Start-Process -NoNewWindow -FilePath "docker-compose" -ArgumentList "-f `"$projectPath\$file`" up -d" -} + Write-Host "🔄 Arrêt du service $file..." -ForegroundColor Yellow + docker-compose -f "$projectPath\$file" down -# Affiche un message une fois tous les containers lancés -Write-Host "Tous les environnements ont été lancés." \ No newline at end of file + Write-Host "🚀 Démarrage du service $file..." -ForegroundColor Cyan + docker-compose -f "$projectPath\$file" up -d + + if ($LASTEXITCODE -eq 0) { + Write-Host "✅ $file démarré avec succès." -ForegroundColor Green + } else { + Write-Host "❌ Échec lors du démarrage de $file." -ForegroundColor Red + } +} +Write-Host "🎉 Tous les services ont été relancés." -ForegroundColor Yellow \ No newline at end of file