BIGTOP-3134. [Puppet] An upgrade to deploy Alluxio 1.8.1 (#443)
authorEvans Ye <evansye@apache.org>
Fri, 18 Jan 2019 09:59:36 +0000 (17:59 +0800)
committerGitHub <noreply@github.com>
Fri, 18 Jan 2019 09:59:36 +0000 (17:59 +0800)
bigtop-deploy/puppet/modules/alluxio/manifests/init.pp
bigtop-deploy/puppet/modules/alluxio/templates/alluxio-env.sh [deleted file]
bigtop-deploy/puppet/modules/alluxio/templates/alluxio-site.properties [new file with mode: 0644]
bigtop-deploy/puppet/modules/alluxio/templates/log4j.properties [deleted file]

index 9c3c10f..6399b80 100644 (file)
@@ -21,26 +21,19 @@ class alluxio {
     }
   }
 
-  class common ($master_host){
+  class common ($master_host, $hdfs_ha = hiera('hadoop::common_hdfs::ha', undef)) {
     package { "alluxio":
       ensure => latest,
     }
 
     exec { "daemon-reload":
-      command => "/usr/bin/systemctl daemon-reload",
+      path => ["/bin", "/usr/bin"],
+      command => "systemctl daemon-reload",
       require => [ Package["alluxio"] ]
     }
 
-    # add logging into /var/log/..
-    file {
-        "/etc/alluxio/conf/log4j.properties":
-        content => template("alluxio/log4j.properties"),
-        require => [Package["alluxio"]]
-    }
-
-    # add alluxio-env.sh to point to alluxio master
-    file { "/etc/alluxio/conf/alluxio-env.sh":
-        content => template("alluxio/alluxio-env.sh"),
+    file { "/etc/alluxio/conf/alluxio-site.properties":
+        content => template("alluxio/alluxio-site.properties"),
         require => [Package["alluxio"]]
     }
   }
@@ -51,14 +44,14 @@ class alluxio {
     exec {
         "alluxio formatting":
            command => "/usr/lib/alluxio/bin/alluxio format",
-           require => [ Package["alluxio"], File["/etc/alluxio/conf/log4j.properties"], File["/etc/alluxio/conf/alluxio-env.sh"] ]
+           require => [ Package["alluxio"], File["/etc/alluxio/conf/alluxio-site.properties"] ]
     }
 
     if ( $fqdn == $alluxio::common::master_host ) {
       service { "alluxio-master":
         ensure => running,
         require => [ Package["alluxio"], Exec["daemon-reload"], Exec["alluxio formatting"] ],
-        subscribe => [ File["/etc/alluxio/conf/log4j.properties"], File["/etc/alluxio/conf/alluxio-env.sh"] ],
+        subscribe => File["/etc/alluxio/conf/alluxio-site.properties"],
         hasrestart => true,
         hasstatus => true,
       }
@@ -77,8 +70,8 @@ class alluxio {
 
     service { "alluxio-worker":
       ensure => running,
-      require => [ Package["alluxio"], Exec["daemon-reload"], File["/etc/alluxio/conf/log4j.properties"], File["/etc/alluxio/conf/alluxio-env.sh"] ],
-      subscribe => [ File["/etc/alluxio/conf/log4j.properties"], File["/etc/alluxio/conf/alluxio-env.sh"] ],
+      require => [ Package["alluxio"], Exec["daemon-reload"], File["/etc/alluxio/conf/alluxio-site.properties"] ],
+      subscribe => File["/etc/alluxio/conf/alluxio-site.properties"],
       hasrestart => true,
       hasstatus => true,
     }
diff --git a/bigtop-deploy/puppet/modules/alluxio/templates/alluxio-env.sh b/bigtop-deploy/puppet/modules/alluxio/templates/alluxio-env.sh
deleted file mode 100755 (executable)
index 27c7fb2..0000000
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This file contains environment variables required to run Alluxio. Copy it as alluxio-env.sh and
-# edit that to configure Alluxio for your site. At a minimum,
-# the following variables should be set:
-#
-# - JAVA_HOME, to point to your JAVA installation
-# - ALLUXIO_MASTER_ADDRESS, to bind the master to a different IP address or hostname
-# - ALLUXIO_UNDERFS_ADDRESS, to set the under filesystem address.
-# - ALLUXIO_WORKER_MEMORY_SIZE, to set how much memory to use (e.g. 1000mb, 2gb) per worker
-# - ALLUXIO_RAM_FOLDER, to set where worker stores in memory data
-# - ALLUXIO_UNDERFS_HDFS_IMPL, to set which HDFS implementation to use (e.g. com.mapr.fs.MapRFileSystem,
-#   org.apache.hadoop.hdfs.DistributedFileSystem)
-
-# The following gives an example:
-
-if [[ `uname -a` == Darwin* ]]; then
-  # Assuming Mac OS X
-  export JAVA_HOME=${JAVA_HOME:-$(/usr/libexec/java_home)}
-  export ALLUXIO_RAM_FOLDER=/Volumes/ramdisk
-  export ALLUXIO_JAVA_OPTS="-Djava.security.krb5.realm= -Djava.security.krb5.kdc="
-else
-  # Assuming Linux
-  if [ -z "$JAVA_HOME" ]; then
-    export JAVA_HOME=/usr/lib/jvm/java-7-oracle
-  fi
-  export ALLUXIO_RAM_FOLDER=/mnt/ramdisk
-fi
-
-export JAVA="$JAVA_HOME/bin/java"
-
-echo "Starting alluxio w/ java = $JAVA "
-
-export ALLUXIO_MASTER_ADDRESS=<%= @master_host %>
-export ALLUXIO_UNDERFS_ADDRESS=$ALLUXIO_HOME/underfs
-#export ALLUXIO_UNDERFS_ADDRESS=hdfs://localhost:9000
-export ALLUXIO_WORKER_MEMORY_SIZE=1GB
-export ALLUXIO_UNDERFS_HDFS_IMPL=org.apache.hadoop.hdfs.DistributedFileSystem
-
-echo "ALLUXIO master => $ALLUXIO_MASTER_ADDRESS "
-
-CONF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-export ALLUXIO_JAVA_OPTS+="
-  -Dlog4j.configuration=file:$CONF_DIR/log4j.properties
-  -Dalluxio.debug=false
-  -Dalluxio.underfs.address=$ALLUXIO_UNDERFS_ADDRESS
-  -Dalluxio.underfs.hdfs.impl=$ALLUXIO_UNDERFS_HDFS_IMPL
-  -Dalluxio.data.folder=$ALLUXIO_UNDERFS_ADDRESS/tmp/alluxio/data
-  -Dalluxio.workers.folder=$ALLUXIO_UNDERFS_ADDRESS/tmp/alluxio/workers
-  -Dalluxio.worker.memory.size=$ALLUXIO_WORKER_MEMORY_SIZE
-  -Dalluxio.worker.data.folder=$ALLUXIO_RAM_FOLDER/alluxioworker/
-  -Dalluxio.master.worker.timeout.ms=60000
-  -Dalluxio.master.hostname=$ALLUXIO_MASTER_ADDRESS
-  -Dalluxio.master.journal.folder=$ALLUXIO_HOME/journal/
-  -Dorg.apache.jasper.compiler.disablejsr199=true
-  -Djava.net.preferIPv4Stack=true
-"
-
-# Master specific parameters. Default to ALLUXIO_JAVA_OPTS.
-export ALLUXIO_MASTER_JAVA_OPTS="$ALLUXIO_JAVA_OPTS"
-
-# Worker specific parameters that will be shared to all workers. Default to ALLUXIO_JAVA_OPTS.
-export ALLUXIO_WORKER_JAVA_OPTS="$ALLUXIO_JAVA_OPTS"
diff --git a/bigtop-deploy/puppet/modules/alluxio/templates/alluxio-site.properties b/bigtop-deploy/puppet/modules/alluxio/templates/alluxio-site.properties
new file mode 100644 (file)
index 0000000..99fefd8
--- /dev/null
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Site specific configuration properties for Alluxio
+# Details about all configuration properties http://www.alluxio.org/documentation/en/Configuration-Settings.html
+
+# Common properties
+alluxio.master.hostname=<%= @master_host %>
+<% if defined?(@hdfs_ha) and @hdfs_ha != "disabled" -%>
+alluxio.underfs.address=<%= scope().call_function('hiera',['hadoop_cluster_node::hadoop_namenode_uri']) %>/underFSStorage
+<% else -%>
+alluxio.underfs.address=hdfs://<%= scope().call_function('hiera',['hadoop::common_hdfs::hadoop_namenode_host']) %>:<%= scope().call_function('hiera', ['hadoop::common_hdfs::hadoop_namenode_port']) %>/underFSStorage
+<% end -%>
+
+# Security properties
+# alluxio.security.authorization.permission.enabled=true
+# alluxio.security.authentication.type=SIMPLE
+
+# Worker properties
+# alluxio.worker.memory.size=1GB
+# alluxio.worker.tieredstore.levels=1
+# alluxio.worker.tieredstore.level0.alias=MEM
+# alluxio.worker.tieredstore.level0.dirs.path=/mnt/ramdisk
+
+# User properties
+# alluxio.user.file.readtype.default=CACHE_PROMOTE
+# alluxio.user.file.writetype.default=MUST_CACHE
+
diff --git a/bigtop-deploy/puppet/modules/alluxio/templates/log4j.properties b/bigtop-deploy/puppet/modules/alluxio/templates/log4j.properties
deleted file mode 100644 (file)
index 00b402e..0000000
+++ /dev/null
@@ -1,77 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# May get overridden by System Property
-alluxio.logger.type=Console
-
-log4j.rootLogger=INFO, ${alluxio.logger.type}
-
-log4j.appender.Console=org.apache.log4j.ConsoleAppender
-log4j.appender.Console.Target=System.out
-log4j.appender.Console.layout=org.apache.log4j.PatternLayout
-log4j.appender.Console.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} (%F:%M) - %m%n
-
-# Appender for Master
-log4j.appender.MASTER_LOGGER=org.apache.log4j.RollingFileAppender
-log4j.appender.MASTER_LOGGER.File=${alluxio.logs.dir}/master.log
-log4j.appender.MASTER_LOGGER.MaxFileSize=10MB
-log4j.appender.MASTER_LOGGER.MaxBackupIndex=100
-log4j.appender.MASTER_LOGGER.layout=org.apache.log4j.PatternLayout
-log4j.appender.MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n
-#log4j.appender.MASTER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-# Appender for Master Access Log
-log4j.appender.MASTER_ACCESS_LOGGER=org.apache.log4j.RollingFileAppender
-log4j.appender.MASTER_ACCESS_LOGGER.File=${alluxio.logs.dir}/master_access.log
-log4j.appender.MASTER_ACCESS_LOGGER.MaxFileSize=10MB
-log4j.appender.MASTER_ACCESS_LOGGER.MaxBackupIndex=100
-log4j.appender.MASTER_ACCESS_LOGGER.layout=org.apache.log4j.PatternLayout
-log4j.appender.MASTER_ACCESS_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n
-#log4j.appender.MASTER_ACCESS_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-# Appender for Workers
-log4j.appender.WORKER_LOGGER=org.apache.log4j.RollingFileAppender
-log4j.appender.WORKER_LOGGER.File=${alluxio.logs.dir}/worker.log
-log4j.appender.WORKER_LOGGER.MaxFileSize=10MB
-log4j.appender.WORKER_LOGGER.MaxBackupIndex=100
-log4j.appender.WORKER_LOGGER.layout=org.apache.log4j.PatternLayout
-log4j.appender.WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n
-#log4j.appender.WORKER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-# Appender for Worker Access Log
-log4j.appender.WORKER_ACCESS_LOGGER=org.apache.log4j.RollingFileAppender
-log4j.appender.WORKER_ACCESS_LOGGER.File=${alluxio.logs.dir}/worker_access.log
-log4j.appender.WORKER_ACCESS_LOGGER.MaxFileSize=10MB
-log4j.appender.WORKER_ACCESS_LOGGER.MaxBackupIndex=100
-log4j.appender.WORKER_ACCESS_LOGGER.layout=org.apache.log4j.PatternLayout
-log4j.appender.WORKER_ACCESS_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n
-#log4j.appender.WORKER_ACCESS_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-# Appender for User
-log4j.appender.USER_LOGGER=org.apache.log4j.RollingFileAppender
-log4j.appender.USER_LOGGER.File=${alluxio.logs.dir}/user.log
-log4j.appender.USER_LOGGER.MaxFileSize=10MB
-log4j.appender.USER_LOGGER.MaxBackupIndex=10
-log4j.appender.USER_LOGGER.layout=org.apache.log4j.PatternLayout
-log4j.appender.USER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n
-#log4j.appender.USER_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-# Appender for Fuse
-log4j.appender.FUSE_LOGGER=org.apache.log4j.RollingFileAppender
-log4j.appender.FUSE_LOGGER.File=${alluxio.logs.dir}/fuse.log
-log4j.appender.FUSE_LOGGER.MaxFileSize=10MB
-log4j.appender.FUSE_LOGGER.MaxBackupIndex=10
-log4j.appender.FUSE_LOGGER.layout=org.apache.log4j.PatternLayout
-log4j.appender.FUSE_LOGGER.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M) - %m%n