bigtop-deploy/juju/hadoop-kafka/bundle.yaml

series: xenial
description: >
  This is a fourteen unit big data cluster that includes Hadoop 2.7 and
  Kafka 0.10 from Apache Bigtop. Use it to store streams of data in HDFS
  for future analysis with MapReduce. It will run on 9 machines in your cloud.
services:
  namenode:
    charm: "cs:xenial/hadoop-namenode-46"
    constraints: "mem=7G root-disk=32G"
    num_units: 1
    annotations:
      gui-x: "500"
      gui-y: "800"
    to:
      - "0"
  resourcemanager:
    charm: "cs:xenial/hadoop-resourcemanager-48"
    constraints: "mem=7G root-disk=32G"
    num_units: 1
    annotations:
      gui-x: "500"
      gui-y: "0"
    to:
      - "0"
  slave:
    charm: "cs:xenial/hadoop-slave-47"
    constraints: "mem=7G root-disk=32G"
    num_units: 3
    annotations:
      gui-x: "0"
      gui-y: "400"
    to:
      - "1"
      - "2"
      - "3"
  plugin:
    charm: "cs:xenial/hadoop-plugin-46"
    annotations:
      gui-x: "1000"
      gui-y: "400"
  client:
    charm: "cs:xenial/hadoop-client-12"
    constraints: "mem=3G"
    num_units: 1
    annotations:
      gui-x: "1250"
      gui-y: "400"
    to:
      - "4"
# TODO: Charm bigtop flume
  flume-hdfs:
    charm: "cs:xenial/apache-flume-hdfs-7"
    num_units: 1
    annotations:
      gui-x: "1500"
      gui-y: "400"
    to:
      - "4"
  zookeeper:
    charm: "cs:xenial/zookeeper-53"
    constraints: "mem=3G root-disk=32G"
    num_units: 3
    annotations:
      gui-x: "500"
      gui-y: "400"
    to:
      - "5"
      - "6"
      - "7"
  kafka:
    charm: "cs:xenial/kafka-51"
    constraints: "mem=3G"
    num_units: 1
    annotations:
      gui-x: "1250"
      gui-y: "800"
    to:
      - "8"
# TODO: Charm bigtop flume
# NOTE: flume-kafka cannot be colocated with flume-hdfs as they both use /etc/flume/conf
  flume-kafka:
    charm: "cs:xenial/apache-flume-kafka-1"
    num_units: 1
    annotations:
      gui-x: "1500"
      gui-y: "800"
    to:
      - "8"
  ganglia:
    charm: "cs:xenial/ganglia-12"
    num_units: 1
    annotations:
      gui-x: "0"
      gui-y: "800"
    to:
      - "4"
  ganglia-node:
    charm: "cs:xenial/ganglia-node-7"
    annotations:
      gui-x: "250"
      gui-y: "400"
  rsyslog:
    charm: "cs:~bigdata-dev/xenial/rsyslog-7"
    num_units: 1
    annotations:
      gui-x: "1000"
      gui-y: "800"
    to:
      - "4"
  rsyslog-forwarder-ha:
    charm: "cs:~bigdata-dev/xenial/rsyslog-forwarder-ha-7"
    annotations:
      gui-x: "750"
      gui-y: "400"
relations:
  - [resourcemanager, namenode]
  - [namenode, slave]
  - [resourcemanager, slave]
  - [plugin, namenode]
  - [plugin, resourcemanager]
  - [client, plugin]
  - [flume-hdfs, plugin]
  - [flume-kafka, flume-hdfs]
  - [flume-kafka, kafka]
  - [kafka, zookeeper]
  - ["ganglia-node:juju-info", "namenode:juju-info"]
  - ["ganglia-node:juju-info", "resourcemanager:juju-info"]
  - ["ganglia-node:juju-info", "slave:juju-info"]
  - ["ganglia-node:juju-info", "kafka:juju-info"]
  - ["ganglia-node:juju-info", "zookeeper:juju-info"]
  - ["ganglia:node", "ganglia-node:node"]
  - ["rsyslog-forwarder-ha:juju-info", "namenode:juju-info"]
  - ["rsyslog-forwarder-ha:juju-info", "resourcemanager:juju-info"]
  - ["rsyslog-forwarder-ha:juju-info", "slave:juju-info"]
  - ["rsyslog-forwarder-ha:juju-info", "kafka:juju-info"]
  - ["rsyslog-forwarder-ha:juju-info", "zookeeper:juju-info"]
  - ["rsyslog:aggregator", "rsyslog-forwarder-ha:syslog"]
machines:
  "0":
    series: "xenial"
    constraints: "mem=7G root-disk=32G"
  "1":
    series: "xenial"
    constraints: "mem=7G root-disk=32G"
  "2":
    series: "xenial"
    constraints: "mem=7G root-disk=32G"
  "3":
    series: "xenial"
    constraints: "mem=7G root-disk=32G"
  "4":
    series: "xenial"
    constraints: "mem=3G"
  "5":
    series: "xenial"
    constraints: "mem=3G root-disk=32G"
  "6":
    series: "xenial"
    constraints: "mem=3G root-disk=32G"
  "7":
    series: "xenial"
    constraints: "mem=3G root-disk=32G"
  "8":
    series: "xenial"
    constraints: "mem=3G"