/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ buildscript { repositories { mavenCentral() } } plugins { id "de.undercouch.download" version "3.2.0" id "org.nosphere.apache.rat" version "0.2.0" } // Generated by http://patorjk.com/software/taag/#p=display&f=Bloody&t=Apache%20Bigtop def final ASCII_bigtop = ''' :aass.. =XS22nai,>__. . =n-- +!!!""^-- .vX> . .)e . ..=-?Xo>, . . . .__aaoZe!`=>.+*a>,.-"!Y1XSSX##ZZXXXXXXXXXXXoXXc .{XXXXXX2*?!!"!"^~--- ...__aa2!^- =dX( .+XXc. ~!1nas,,.---~~^""!"!!!?YSXXXXX2+ -"YSXXXo=. ._=sssaaav1!!~- ._aXXe` )SXo>. -~"?Yoouass_s,, _vXXXX2}~ -{XXZoai%%*XXSSSX>.. . - ---=2XXX2^-"|||}"--~{ZXXX1-- .:XXXXo; . )XXXX2` . =XXXXZc nXXXX> =XXXXe..__s=>_...)XXXX1 . .:SXXXo; .)XXXX2.. =XXXXosummmmBmma,)ZXXX1 :XXXX2; )XXXXX. . :XXXXXc . . =oZXXXe; .:test task only * in if the project has smoke.tests properties set. * This is done to avoid running cluster smoke tests during the normal life-cycle * of the project development */ def checkClusterTestProjects = { property -> def suiteName = property.replaceAll('.', '-') FileTree fTree = fileTree(dir: "bigtop-tests/$suiteName", include: '*/build.gradle') fTree.each() { smokeProject -> def parent = smokeProject.getParentFile().name project (":bigtop-tests:$suiteName:$parent") { test { onlyIf {project.hasProperty(suiteName)} } } } } def TESTARTIFACTS_GROUP = 'test artifacts' def DEVENV_GROUP = 'development tools' def DEPLOY_GROUP = 'deployment' def DOCKERBUILD_GROUP = 'docker build' // All packaging logic is separated into its own build module apply from: 'packages.gradle' // Wrapping all release logic to simplify the release process apply from: 'release.gradle' task toolchain(type:Exec, description: 'Setup dev. env via toolchain; Requires: Puppet, sudo', group: DEVENV_GROUP) { def command = ['sudo', 'puppet', 'apply', '-d'] def version = "sudo puppet --version".execute().text if ('3.7' <= version && version < '4') { command.addAll(['--parser', 'future']) } command.addAll(["--modulepath=${projectDir.absolutePath}:/etc/puppet/modules:/usr/share/puppet/modules:/etc/puppetlabs/code/modules", '-e', 'include bigtop_toolchain::installer']) workingDir '.' commandLine command } task "toolchain-puppetmodules"(type:Exec, description: 'Setup puppet modules via toolchain; Requires: Puppet, sudo', group: DEVENV_GROUP) { def command = [ 'sudo', 'puppet', 'apply', '-d', "--modulepath=${projectDir.absolutePath}", '-e', 'include bigtop_toolchain::puppet_modules' ] workingDir '.' commandLine command } task "toolchain-devtools"(type:Exec, description: 'Setup additional dev. tools like Groovy SDK via toolchain; Requires: Puppet, sudo', group: DEVENV_GROUP) { def command = [ 'sudo', 'puppet', 'apply', '-d', "--modulepath=${projectDir.absolutePath}", '-e', 'include bigtop_toolchain::development_tools' ] workingDir '.' commandLine command } task "docker-provisioner"(type:Exec, description: ''' Provision a Bigtop stack cluster on Docker container(s). Default to CentOS and 1 node. Properties: -Pconfig=[CONFIG_FILE] (located under provisioner/docker/) -Penable_local_repo -Pimage=[DOCKER_IMAGE] (overwrites -POS and -Pprefix) -Pmemory=[4g|8g|...] -Pnum_instances=[NUM_INSTANCES] -Pnexus=[NEXUS_URL] (NEXUS_URL is optional) -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3] -Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...] -Prepository=[REPO_URL] -Prun_smoke_tests (run test components defined in config file) -Psmoke_tests=[COMPONENTS] -Pstack=[COMPONENTS] For detailed explaination, please refer to the helper message: $ provisioner/docker/docker-hadoop.sh --help Examples: Create a 3 node cluster: $ ./gradlew -Pnum_instances=3 docker-provisioner Run smoke tests on 3 node cluster: $ ./gradlew -Pnum_instances=3 -Prun_smoke_tests docker-provisioner Specify custom configuration file: $ ./gradlew -Pconfig=config_debian-9.yaml docker-provisioner ''', group: DEPLOY_GROUP) { def _config = project.hasProperty("config") ? config : "config.yaml" def _num_instances = project.hasProperty("num_instances") ? num_instances : 1 def _OS = project.hasProperty("OS") ? OS : 'centos-7' def _prefix = project.hasProperty("prefix") ? prefix : 'trunk' def command = [ './docker-hadoop.sh', '-C', _config, '--create', _num_instances, ] if ( project.hasProperty("stack") ) { command.add('--stack') command.add("${stack}") } if ( project.hasProperty("run_smoke_tests") ) { command.add("--smoke-tests") } if ( project.hasProperty("smoke_tests") ) { command.add('--smoke-tests') command.add("${smoke_tests}") } if ( project.hasProperty("nexus") ) { command.add('--nexus') command.add("${nexus}") } if ( project.hasProperty("enable_local_repo") ) { command.add('--enable-local-repo') } if (project.hasProperty("OS") || project.hasProperty("prefix")) { command.add('--image') command.add("bigtop/puppet:${_prefix}-${_OS}") } if (project.hasProperty("image")) { command.add('--image') command.add("${image}") } if (project.hasProperty("repository")) { command.add('--repo') command.add("${repository}") } if (project.hasProperty("memory")) { command.add('--memory') command.add("${memory}") } workingDir 'provisioner/docker' commandLine command } task "docker-provisioner-ssh"(type:Exec, description: 'Show command to get in to the provisioned container', group: DEPLOY_GROUP) { workingDir 'provisioner/docker' def message = "To get in to the container, do: \n\ndocker exec -ti bash \n\nYou can get container name by ./gradlew docker-provisioner-status" def command = [ '/bin/echo', message ] commandLine command } task "docker-provisioner-status"(type:Exec, description: 'Show status of Bigtop Docker cluster', group: DEPLOY_GROUP) { def command = [ './docker-hadoop.sh', '--list', ] workingDir 'provisioner/docker' commandLine command } task "docker-provisioner-destroy"(type:Exec, description: 'Destroy provisioned Bigtop Docker cluster', group: DEPLOY_GROUP) { def command = [ './docker-hadoop.sh', '--destroy' ] workingDir 'provisioner/docker' commandLine command } task installTopLevel(type:Exec) { workingDir "." commandLine 'mvn clean install -f pom.xml'.split(" ") } task installiTest(dependsOn: installTopLevel, type:Exec) { workingDir "." commandLine 'mvn clean install -f bigtop-test-framework/pom.xml -DskipTests'.split(" ") } task installTestArtifacts(dependsOn: installiTest, type:Exec) { workingDir "." commandLine 'mvn clean install -f bigtop-tests/test-artifacts/pom.xml'.split(" ") } task installConf(type:Exec) { workingDir "." commandLine 'mvn clean install -f bigtop-tests/test-execution/conf/pom.xml'.split(" ") } task installCommon(type:Exec) { workingDir "." commandLine 'mvn clean install -f bigtop-tests/test-execution/common/pom.xml'.split(" ") } task installAllLocalArtifacts ( description: "Prepare and locally install all test artifacts", group: TESTARTIFACTS_GROUP) { } /** * Allows user to specify which artifacts to install by dynamically generating tasks. */ def artifactToInstall = { def final BASE_DIR = projectDir.absolutePath def final TEST_DIR = "$BASE_DIR/bigtop-tests/test-artifacts" def project = new XmlSlurper().parse("$TEST_DIR/pom.xml") project.modules.module.each { artifact -> task "install-${artifact}" (description: "Installs ${artifact} artifact with Maven", group: TESTARTIFACTS_GROUP, dependsOn: installiTest ) doLast { def final PATH = "${TEST_DIR}/$artifact/pom.xml" def final WRAPPER = "mvn clean install -f " + PATH exec { workingDir '.' commandLine WRAPPER.split(" ") } } } } def _NEXUS_URL = project.hasProperty("NEXUS_URL") ? NEXUS_URL : "http://localhost:8081/nexus" /** * Programatically create configure-nexus-* gradle tasks using a name, url and snapshot variable. */ def generate_nexus_tasks = { name, url, snapshot -> task "configure-nexus-${name}"(type: Exec) { def req = [ data : [ id: name, name: name, repoType: "proxy", repoPolicy : snapshot, browseable: true, "indexable": true, "notFoundCacheTTL": 1440, "artifactMaxAge": 1440, "metadataMaxAge": 1440, "itemMaxAge": 1440, "provider": "maven2", "providerRole": "org.sonatype.nexus.proxy.repository.Repository", "downloadRemoteIndexes": true, "autoBlockActive": true, "fileTypeValidation": true, "exposed": true, "checksumPolicy": "WARN", "remoteStorage": ["remoteStorageUrl": url, "authentication": null, "connectionSettings": null]]] def root = new groovy.json.JsonBuilder(req) def _NEXUS_USERPASS = project.hasProperty("NEXUS_USERPASS") ? NEXUS_USERPASS : "admin:admin123" File.createTempFile("temp",".tmp").with { deleteOnExit() write root.toString() workingDir '.' commandLine "curl", "-o", "/dev/null", "-s", "-w", "%{http_code}", "-X", "POST", "-d", "@$absolutePath", "--header", "Content-Type: application/json", "-u", "$_NEXUS_USERPASS", "$_NEXUS_URL/service/local/repositories" standardOutput = new ByteArrayOutputStream() doLast { def httpCode = standardOutput.toString() switch (httpCode) { case "200": println "Nexus proxy ${name} successfully configured" break case "201": println "Got status 201 from nexus" break case "400": println "Skipped because Nexus proxy ${name} already configured" break default: throw new GradleException("Failed to configure Nexus proxy ${name} with http code ${httpCode} returned. " + "Run with --info option to see the executed command") break } } } } } def repos = [ [ "name": "central", "url" : "https://repo.maven.apache.org/maven2/", 'snapshot':'RELEASE'], [ "name": "conjars", "url" : "http://conjars.org", 'snapshot':'RELEASE'], [ "name": "repository.jboss.org", "url": "http://repository.jboss.org/nexus/content/groups/public/", 'snapshot': 'RELEASE'], [ "name": "apache.snapshots.https", "url": "https://repository.apache.org/content/repositories/snapshots", "snapshot": 'SNAPSHOT'], [ "name": "apache.snapshots", "url": "https://repository.apache.org/content/repositories/snapshots", "snapshot": 'SNAPSHOT'], [ "name": "spring-milestones", "url": "http://repo.spring.io/milestone", "snapshot": 'RELEASE'], [ "name": "maven2-repository.atlassian", "url": "https://maven.atlassian.com/repository/public", "snapshot": 'RELEASE'] ] /** * For all repos in repos list, create configure-nexus-* gradle tasks */ repos.each { r-> generate_nexus_tasks( r.name, r.url, r.snapshot) } /** * Gradle task that updates the public group in Nexus. 'reposInGroup' is a JSON array containing the repo ids. * Because of the 'dependsOn'-attribute, the task will always be executed after all configure-nexus-* tasks. */ task "configure-nexus-public-group"(type: Exec, dependsOn: [ tasks.findAll { alltask -> alltask.name.startsWith("configure-nexus-")}*.name ]) { def reposInGroup = [] repos.each {r-> reposInGroup.add (["id": r.name]) } def req = [ data : [ id: "public", name: "Public Repositories", "format":"maven2","exposed":true,"provider":"maven2", "repositories": reposInGroup ] ] def root = new groovy.json.JsonBuilder(req) def _NEXUS_USERPASS = project.hasProperty("NEXUS_USERPASS") ? NEXUS_USERPASS : "admin:admin123" File.createTempFile("temp",".tmp").with { deleteOnExit() write root.toString() workingDir '.' commandLine "curl", "-o", "/dev/null", "-s", "-w", "%{http_code}", "-X", "PUT", "-d", "@$absolutePath", "--header", "Content-Type: application/json", "-u", "$_NEXUS_USERPASS", "$_NEXUS_URL/service/local/repo_groups/public" standardOutput = new ByteArrayOutputStream() doLast { def httpCode = standardOutput.toString() switch (httpCode) { case "200": println "Nexus public group successfully configured" break case "201": println "Got status 201 from nexus" break case "400": println "Skipped because Nexus proxy was already configured" break default: throw new GradleException("Failed to configure Nexus proxy ${name} with http code ${httpCode} returned. " + "Run with --info option to see the executed command") break } } } } /** * 'Entry point' gradle task to configure Nexus. It will write a maven settings.xml file containing a single refrence to the * Nexus 'public' group. The mirrorsOf-element contains a comma-separated string with repositoryIds (from the repos-variable). * This makes sure that the build won't fail whenever the repos-variable gets outdated. * The 'dependsOn' attribute makes sure all the above 'configure-nexus-*' tasks are executed first (in the right order). */ task "configure-nexus"(dependsOn: [ tasks.findAll { alltask -> alltask.name.startsWith("configure-nexus-")}*.name ], description: 'configure build to use a sonatype nexus server.\n' + ' Use -PNEXUS_URL=http://server:8081 to overwrite default server http://localhost:8081/nexus\n' + ' Use -PNEXUS_USERPASS=user:passwd to overwrite default username passwd admin:admin123') doLast { def m2Dir = System.getProperty("user.home") + "/.m2" mkdir(m2Dir) def m2Writer = new File(m2Dir + "/settings.xml") m2Writer.text = "" def repoString = "" repos.each {r-> repoString +=r.name + "," } m2Writer.append("local$_NEXUS_URL/content/groups/public/$repoString") m2Writer.append("") def gradleDir = System.getProperty("user.home") + "/.gradle/init.d" mkdir(gradleDir) def gradleWriter = new File(gradleDir + "/bigtop_nexus.gradle") gradleWriter.text = """ allprojects { repositories { maven { url "$_NEXUS_URL/content/groups/public/" } } } """ } task "bigtop-puppet"(type:Exec, description: ''' Build bigtop/puppet images Usage: $ ./gradlew -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3] -Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...] bigtop-puppet Example: $ ./gradlew -POS=debian-9 -Pprefix=1.0.0 bigtop-puppet The built image name: bigtop/puppet:1.0.0-debian-9 ''', group: DOCKERBUILD_GROUP) { def _prefix = project.hasProperty("prefix") ? prefix : "trunk" def _OS = project.hasProperty("OS") ? OS : "centos-7" def command = [ './build.sh', _prefix + '-' + _OS, ] workingDir './docker/bigtop-puppet' commandLine command } task "bigtop-slaves"(type:Exec, description: ''' Build bigtop/slaves images Usage: $ ./gradlew -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3] -Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...] bigtop-slaves Example: $ ./gradlew -POS=debian-9 -Pprefix=1.0.0 bigtop-slaves The built image name: bigtop/slaves:1.0.0-debian-9 ''', group: DOCKERBUILD_GROUP) { def _prefix = project.hasProperty("prefix") ? prefix : "trunk" def _OS = project.hasProperty("OS") ? OS : "centos-7" def command = [ './build.sh', _prefix + '-' + _OS, ] workingDir './docker/bigtop-slaves' commandLine command } project.afterEvaluate { checkClusterTestProjects("smoke.tests") artifactToInstall(dependsOn: [installTopLevel, installCommon, installConf, installiTest]) } installAllLocalArtifacts.dependsOn installTopLevel, installCommon, installConf, installiTest, installTestArtifacts help.doFirst { println ASCII_bigtop }