FullStack_BigDataAnalytics_Project/docker-compose.yml at main · zee404-code/FullStack_BigDataAnalytics_Project · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
version: "3.9"

services:
  mongo:
    image: mongo:4.4
    container_name: mongo
    environment:
      MONGO_INITDB_ROOT_USERNAME: ${MONGO_INITDB_ROOT_USERNAME}
      MONGO_INITDB_ROOT_PASSWORD: ${MONGO_INITDB_ROOT_PASSWORD}
    ports:
      - "27017:27017"
    volumes:
      - mongo_data:/data/db

  redis:
    image: redis:7
    container_name: redis
    ports:
      - "6379:6379"

  # Hadoop HDFS (archive + metadata)
  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
    container_name: namenode
    environment:
      - CLUSTER_NAME=flashsale
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
    ports:
      - "9870:9870"   # NameNode UI
      - "8020:8020"
    volumes:
      - namenode_data:/hadoop/dfs/name

  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    container_name: datanode
    environment:
      - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
    depends_on:
      - namenode
    ports:
      - "9864:9864"
    volumes:
      - datanode_data:/hadoop/dfs/data

  # Spark (processing/OLAP)
  spark-master:
    image: apache/spark:3.5.1
    container_name: spark-master
    command: >
      /opt/spark/bin/spark-class
      org.apache.spark.deploy.master.Master
    ports:
      - "7077:7077"
      - "8080:8080"
    volumes:
      - ./spark/jobs:/opt/spark/jobs

  spark-worker:
    image: apache/spark:3.5.1
    container_name: spark-worker
    command: >
      /opt/spark/bin/spark-class
      org.apache.spark.deploy.worker.Worker
      spark://spark-master:7077
    depends_on:
      - spark-master

  # Airflow (orchestration)
  airflow:
    build: ./airflow
    container_name: airflow
    user: "50000:0"
    command: airflow standalone
    restart: unless-stopped
    environment:
      - AIRFLOW__CORE__EXECUTOR=SequentialExecutor
      - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=sqlite:////opt/airflow/airflow.db
      - AIRFLOW__CORE__LOAD_EXAMPLES=False
      - AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
      - MONGO_URI=mongodb://mongo:27017/flashsale
    ports:
      - "8088:8080"
    volumes:
      - ./airflow/dags:/opt/airflow/dags
      - ./airflow/logs:/opt/airflow/logs
      - ./airflow/plugins:/opt/airflow/plugins
      - ./spark/jobs:/opt/spark/jobs
  # Streaming statistical generator
  generator:
    build: ./generator
    container_name: generator
    environment:
      - MONGO_URI=mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017/${MONGO_DB}?authSource=admin
      - ORDERS_PER_MIN_BASE=${ORDERS_PER_MIN_BASE}
      - LIVE_WINDOW_MINUTES=${LIVE_WINDOW_MINUTES}
    depends_on:
      - mongo

  # Dashboard (Streamlit)
  dashboard:
    build: ./dashboard
    container_name: dashboard
    environment:
      - MONGO_URI=mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017/${MONGO_DB}?authSource=admin
      - REDIS_HOST=redis
    ports:
      - "8501:8501"
    depends_on:
      - mongo
      - redis

volumes:
  mongo_data:
  namenode_data:
  datanode_data: