# limitations under the License.
#
+function exit_with_usage {
+ cat << EOF
+
+ USAGE: update-doc.sh [spark] [flink]
+
+ This script generates the documentation for the individual Bahir (Spark) and
+ Bahir-Flink modules from the contents of the respective module's README.md
+ files as found in the Bahir and Bahir-Flink source repositories.
+
+ bahir (source repo) bahir-website
+ . └─site
+ . └─docs
+ . └─spark
+ ├─sql-streaming-mqtt └─current
+ │ └─README.md ─────> ├─spark-sql-streaming-mqtt.md
+ ├─streaming-akka │
+ │ └─README.md ─────> ├─spark-streaming-akka.md
+ ├─streaming-mqtt │
+ │ └─README.md ─────> ├─spark-streaming-mqtt.md
+ ├─streaming-twitter │
+ │ └─README.md ─────> ├─spark-streaming-twitter.md
+ └─streaming-zeromq │
+ └─README.md ─────> └─spark-streaming-zeromq.md
+
+ bahir-flink (source repo) bahir-website
+ . └─site
+ . └─docs
+ . └─flink
+ ├─sql-streaming-mqtt └─current
+ │ └─README.md ─────> ├─flink-streaming-activemq.md
+ ├─streaming-akka │
+ │ └─README.md ─────> ├─flink-streaming-akka.md
+ ├─streaming-mqtt │
+ │ └─README.md ─────> ├─flink-streaming-flume.md
+ ├─streaming-twitter │
+ │ └─README.md ─────> ├─flink-streaming-netty.md
+ └─streaming-zeromq │
+ └─README.md ─────> └─flink-streaming-redis.md
+
+
+ Page header with license text comes from the respective template files
+ under:
+ site/docs/spark/templates
+ site/docs/flink/templates
+
+EOF
+ exit 1
+}
+
+if [[ "$@" =~ "-h" ]]; then
+ exit_with_usage
+fi
+
+REPOS="${@:-'spark flink'}"
+
set -e
-BASE_DIR=$(pwd)
-WEBSITE_DOC_DIR=$BASE_DIR/site/docs/spark/current
-BAHIR_SOURCE_DIR=$BASE_DIR/target/bahir
+BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+SPARK_WEBSITE_TEMPLATES_DIR=site/docs/spark/templates
+SPARK_WEBSITE_DOC_DIR=site/docs/spark/current
+SPARK_REPO_NAME=bahir
+SPARK_BAHIR_SOURCE_DIR=target/$SPARK_REPO_NAME
+
+FLINK_WEBSITE_TEMPLATES_DIR=site/docs/flink/templates
+FLINK_WEBSITE_DOC_DIR=site/docs/flink/current
+FLINK_REPO_NAME=bahir-flink
+FLINK_BAHIR_SOURCE_DIR=target/$FLINK_REPO_NAME
+
+GIT_REF=${GIT_REF:-master}
function checkout_code {
# Checkout code
+ cd "$BASE_DIR" # make sure we're in the base dir
+
rm -rf target
mkdir target
cd target
- rm -rf bahir
- git clone https://git-wip-us.apache.org/repos/asf/bahir.git
- cd bahir
- git checkout $GIT_REF
- git_hash=`git rev-parse --short HEAD`
- echo "Checked out Bahir git hash $git_hash"
+ git clone "https://gitbox.apache.org/repos/asf/$1.git" --quiet
+ cd "$1"
+ git checkout "$GIT_REF"
+ git_hash=$(git rev-parse --short HEAD)
+ echo "Checked out $1 git hash $git_hash"
git clean -d -f -x
- cd "$BASE_DIR" #return to base dir
+ cd "$BASE_DIR" # return to base dir
}
-checkout_code
+function update_docs {
+ WEBSITE_TEMPLATES_DIR=$1
+ WEBSITE_DOC_DIR=$2
+ SOURCE_DIR=$3
+ shift 3
+
+ [ ! -d "$WEBSITE_DOC_DIR" ] && mkdir "$WEBSITE_DOC_DIR"
+
+ echo "Updating documents ..."
+
+ while [ $# -ne 0 ]; do
+ echo " $WEBSITE_DOC_DIR/$1.md <-- $SOURCE_DIR/$2/README.md"
+ rm -f "$WEBSITE_DOC_DIR/$1.md"
+ cp "$WEBSITE_TEMPLATES_DIR/$1.template" "$WEBSITE_DOC_DIR/$1.md"
+ cat "$SOURCE_DIR/$2/README.md" >> "$WEBSITE_DOC_DIR/$1.md"
+ shift 2
+ done
+}
+
+function check_version_strings {
+ if grep -q -r "[0-9]-SNAPSHOT" "$1"/*.md ; then
+ echo
+ echo "TODO: Replace '...-SNAPSHOT' version strings:"
+ echo
+ grep -r -n "[0-9]-SNAPSHOT" "$1"/*.md | sed 's/^/ /' | grep --color "[0-9.]*-SNAPSHOT"
+ echo
+ echo "i.e. to replace '2.1.0-SNAPSHOT' with '2.0.2' run the following command:"
+ echo
+ echo " perl -i -pe 's/2.1.0-SNAPSHOT/2.0.2/g' $1/*.md"
+ echo
+ else
+ echo
+ echo "Generated files:"
+ echo
+ ls "$1"/*.md | xargs -n1 | sed -e 's|'"$(pwd -P)"/'||g'
+ fi
+}
-rm $WEBSITE_DOC_DIR/spark*.md
+function update_spark {
+ echo
+ echo "================= Updating Apache Spark Extension documents ================="
+ echo
-cp $WEBSITE_DOC_DIR/spark-sql-streaming-mqtt.template $WEBSITE_DOC_DIR/spark-sql-streaming-mqtt.md
-cat $BAHIR_SOURCE_DIR/sql-streaming-mqtt/README.md >> $WEBSITE_DOC_DIR/spark-sql-streaming-mqtt.md
+ checkout_code $SPARK_REPO_NAME
-cp $WEBSITE_DOC_DIR/spark-streaming-akka.template $WEBSITE_DOC_DIR/spark-streaming-akka.md
-cat $BAHIR_SOURCE_DIR/streaming-akka/README.md >> $WEBSITE_DOC_DIR/spark-streaming-akka.md
+ update_docs "$SPARK_WEBSITE_TEMPLATES_DIR" \
+ "$SPARK_WEBSITE_DOC_DIR" "$SPARK_BAHIR_SOURCE_DIR" \
+ spark-sql-cloudant sql-cloudant \
+ spark-sql-streaming-akka sql-streaming-akka \
+ spark-sql-streaming-mqtt sql-streaming-mqtt \
+ spark-streaming-akka streaming-akka \
+ spark-streaming-mqtt streaming-mqtt \
+ spark-streaming-pubsub streaming-pubsub \
+ spark-streaming-pubnub streaming-pubnub \
+ spark-streaming-twitter streaming-twitter \
+ spark-streaming-zeromq streaming-zeromq
-cp $WEBSITE_DOC_DIR/spark-streaming-mqtt.template $WEBSITE_DOC_DIR/spark-streaming-mqtt.md
-cat $BAHIR_SOURCE_DIR/streaming-mqtt/README.md >> $WEBSITE_DOC_DIR/spark-streaming-mqtt.md
+ check_version_strings "$SPARK_WEBSITE_DOC_DIR"
+}
+
+function update_flink {
+ echo
+ echo "================= Updating Apache Flink Extension documents ================="
+ echo
+
+ checkout_code $FLINK_REPO_NAME
+
+ update_docs "$FLINK_WEBSITE_TEMPLATES_DIR" \
+ "$FLINK_WEBSITE_DOC_DIR" "$FLINK_BAHIR_SOURCE_DIR" \
+ flink-streaming-activemq flink-connector-activemq \
+ flink-streaming-akka flink-connector-akka \
+ flink-streaming-flume flink-connector-flume \
+ flink-streaming-influxdb flink-connector-influxdb \
+ flink-streaming-influxdb2 flink-connector-influxdb2 \
+ flink-streaming-kudu flink-connector-kudu \
+ flink-streaming-netty flink-connector-netty \
+ flink-streaming-pinot flink-connector-pinot \
+ flink-streaming-redis flink-connector-redis
+
+ check_version_strings "$FLINK_WEBSITE_DOC_DIR"
+}
-cp $WEBSITE_DOC_DIR/spark-streaming-twitter.template $WEBSITE_DOC_DIR/spark-streaming-twitter.md
-cat $BAHIR_SOURCE_DIR/streaming-twitter/README.md >> $WEBSITE_DOC_DIR/spark-streaming-twitter.md
-cp $WEBSITE_DOC_DIR/spark-streaming-zeromq.template $WEBSITE_DOC_DIR/spark-streaming-zeromq.md
-cat $BAHIR_SOURCE_DIR/streaming-zeromq/README.md >> $WEBSITE_DOC_DIR/spark-streaming-zeromq.md
+[[ "$REPOS" =~ "spark" ]] && update_spark
+[[ "$REPOS" =~ "flink" ]] && update_flink
set +e
projects / bahir-website.git / blobdiff