用docker方式部署hadoop集群

由于用docker方式部署hadoop集群的资料不多，自己也折腾了很久，所以记录一下 这里参考了

uhopper / hadoop-docker — Bitbucket https://bitbucket.org/uhopper/hadoop-docker

因为我测试机是几台机器组成的swarm mode集群 好多配置按照文中配置行不通，所以改了一下

version: '3.4'
services:
    namenode:
      image: uhopper/hadoop-namenode:2.8.1
      hostname: namenode
      networks: 
        - hadoop
      volumes:
        - /var/lib/docker/volumes/hadoop/namenode:/hadoop/dfs/name
      environment:
        - CLUSTER_NAME=datanode1
        - CLUSTER_NAME=datanode2
        - CLUSTER_NAME=datanode3
      ports:
        - "8020:8020"
        - "50070:50070"
    datanode1:
      image: uhopper/hadoop-datanode:2.8.1
      hostname: datanode1
      networks: 
        - hadoop
      volumes:
        - /var/lib/docker/volumes/hadoop/datanode1:/hadoop/dfs/data
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
        
    datanode2:
      image: uhopper/hadoop-datanode:2.8.1
      hostname: datanode2
      networks: 
        - hadoop
      volumes:
        - /var/lib/docker/volumes/hadoop/datanode2:/hadoop/dfs/data
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020

    datanode3:
      image: uhopper/hadoop-datanode:2.8.1
      hostname: datanode3
      networks: 
        - hadoop
      volumes:
        - /var/lib/docker/volumes/hadoop/datanode3:/hadoop/dfs/data
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
    
    resourcemanager:
      image: uhopper/hadoop-resourcemanager:2.8.1
      hostname: resourcemanager
      networks: 
        - hadoop
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
        - YARN_CONF_yarn_log___aggregation___enable=true
      ports:
        - "8088:8088"
        - "8030:8030"
        - "8031:8031"
        - "8032:8032"
        - "8033:8033"
    nodemanager:
      image: uhopper/hadoop-nodemanager:2.8.1
      hostname: nodemanager
      networks: 
        - hadoop
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
        - YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
        - YARN_CONF_yarn_log___aggregation___enable=true
        - YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs
      ports:
        - "46655:46655"
        - "8040:8040"
        - "8042:8042"
networks:
    hadoop:

注意

涉及到volumes挂载到宿主机目录，这里需要预先手动创建一下，不然可能会报错

参考文档

https://www.jianshu.com/p/6bcd72083c08