74cec3e533d8b8e07f8309cf8ce6c0467f92adbf
[hive.git] / ql / src / java / org / apache / hadoop / hive / ql / exec / ExplainTask.java
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 package org.apache.hadoop.hive.ql.exec;
20
21 import static org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME;
22
23 import org.apache.commons.lang3.tuple.ImmutablePair;
24
25 import java.io.OutputStream;
26 import java.io.PrintStream;
27 import java.io.Serializable;
28 import java.lang.annotation.Annotation;
29 import java.lang.reflect.InvocationTargetException;
30 import java.lang.reflect.Method;
31 import java.util.ArrayList;
32 import java.util.Arrays;
33 import java.util.Collection;
34 import java.util.Comparator;
35 import java.util.HashMap;
36 import java.util.HashSet;
37 import java.util.LinkedHashMap;
38 import java.util.List;
39 import java.util.Map;
40 import java.util.Stack;
41 import java.util.Map.Entry;
42 import java.util.Set;
43 import java.util.TreeMap;
44
45 import org.apache.hadoop.fs.Path;
46 import org.apache.hadoop.hive.common.ObjectPair;
47 import org.apache.hadoop.hive.common.jsonexplain.JsonParser;
48 import org.apache.hadoop.hive.common.jsonexplain.JsonParserFactory;
49 import org.apache.hadoop.hive.conf.HiveConf;
50 import org.apache.hadoop.hive.conf.Validator.StringSet;
51 import org.apache.hadoop.hive.metastore.api.FieldSchema;
52 import org.apache.hadoop.hive.ql.Driver;
53 import org.apache.hadoop.hive.ql.DriverContext;
54 import org.apache.hadoop.hive.ql.exec.spark.SparkTask;
55 import org.apache.hadoop.hive.ql.exec.tez.TezTask;
56 import org.apache.hadoop.hive.ql.exec.vector.VectorGroupByOperator;
57 import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext;
58 import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
59 import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
60 import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
61 import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
62 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
63 import org.apache.hadoop.hive.ql.io.AcidUtils;
64 import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
65 import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
66 import org.apache.hadoop.hive.ql.lib.Dispatcher;
67 import org.apache.hadoop.hive.ql.lib.GraphWalker;
68 import org.apache.hadoop.hive.ql.lib.Node;
69 import org.apache.hadoop.hive.ql.lib.NodeProcessor;
70 import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
71 import org.apache.hadoop.hive.ql.lib.Rule;
72 import org.apache.hadoop.hive.ql.metadata.Table;
73 import org.apache.hadoop.hive.ql.optimizer.physical.StageIDsRearranger;
74 import org.apache.hadoop.hive.ql.optimizer.physical.Vectorizer;
75 import org.apache.hadoop.hive.ql.optimizer.physical.VectorizerReason;
76 import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
77 import org.apache.hadoop.hive.ql.parse.ExplainConfiguration.VectorizationDetailLevel;
78 import org.apache.hadoop.hive.ql.parse.SemanticException;
79 import org.apache.hadoop.hive.ql.plan.BaseWork;
80 import org.apache.hadoop.hive.ql.plan.Explain;
81 import org.apache.hadoop.hive.ql.plan.Explain.Level;
82 import org.apache.hadoop.hive.ql.plan.Explain.Vectorization;
83 import org.apache.hadoop.hive.ql.plan.AggregationDesc;
84 import org.apache.hadoop.hive.ql.plan.ExplainWork;
85 import org.apache.hadoop.hive.ql.plan.GroupByDesc;
86 import org.apache.hadoop.hive.ql.plan.HiveOperation;
87 import org.apache.hadoop.hive.ql.plan.MapredWork;
88 import org.apache.hadoop.hive.ql.plan.MapWork;
89 import org.apache.hadoop.hive.ql.plan.ReduceWork;
90 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
91 import org.apache.hadoop.hive.ql.plan.SparkWork;
92 import org.apache.hadoop.hive.ql.plan.TableDesc;
93 import org.apache.hadoop.hive.ql.plan.TezWork;
94 import org.apache.hadoop.hive.ql.plan.VectorReduceSinkInfo;
95 import org.apache.hadoop.hive.ql.plan.VectorReduceSinkDesc;
96 import org.apache.hadoop.hive.ql.plan.VectorGroupByDesc;
97 import org.apache.hadoop.hive.ql.plan.api.StageType;
98 import org.apache.hadoop.hive.ql.security.authorization.AuthorizationFactory;
99 import org.apache.hadoop.hive.ql.session.SessionState;
100 import org.apache.hadoop.hive.serde2.Deserializer;
101 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
102 import org.apache.hadoop.io.IOUtils;
103 import org.apache.hadoop.util.StringUtils;
104 import org.apache.hive.common.util.AnnotationUtils;
105 import org.json.JSONArray;
106 import org.json.JSONException;
107 import org.json.JSONObject;
108 import org.slf4j.Logger;
109 import org.slf4j.LoggerFactory;
110
111 /**
112 * ExplainTask implementation.
113 *
114 **/
115 public class ExplainTask extends Task<ExplainWork> implements Serializable {
116 private static final long serialVersionUID = 1L;
117 public static final String EXPL_COLUMN_NAME = "Explain";
118 private final Set<Operator<?>> visitedOps = new HashSet<Operator<?>>();
119 private boolean isLogical = false;
120 protected final Logger LOG;
121
122 public ExplainTask() {
123 super();
124 LOG = LoggerFactory.getLogger(this.getClass().getName());
125 }
126
127 /*
128 * Below method returns the dependencies for the passed in query to EXPLAIN.
129 * The dependencies are the set of input tables and partitions, and are
130 * provided back as JSON output for the EXPLAIN command.
131 * Example output:
132 * {"input_tables":[{"tablename": "default@test_sambavi_v1", "tabletype": "TABLE"}],
133 * "input partitions":["default@srcpart@ds=2008-04-08/hr=11"]}
134 */
135 private static JSONObject getJSONDependencies(ExplainWork work)
136 throws Exception {
137 assert(work.getDependency());
138
139 JSONObject outJSONObject = new JSONObject(new LinkedHashMap<>());
140 List<Map<String, String>> inputTableInfo = new ArrayList<Map<String, String>>();
141 List<Map<String, String>> inputPartitionInfo = new ArrayList<Map<String, String>>();
142 for (ReadEntity input: work.getInputs()) {
143 switch (input.getType()) {
144 case TABLE:
145 Table table = input.getTable();
146 Map<String, String> tableInfo = new LinkedHashMap<String, String>();
147 tableInfo.put("tablename", table.getCompleteName());
148 tableInfo.put("tabletype", table.getTableType().toString());
149 if ((input.getParents() != null) && (!input.getParents().isEmpty())) {
150 tableInfo.put("tableParents", input.getParents().toString());
151 }
152 inputTableInfo.add(tableInfo);
153 break;
154 case PARTITION:
155 Map<String, String> partitionInfo = new HashMap<String, String>();
156 partitionInfo.put("partitionName", input.getPartition().getCompleteName());
157 if ((input.getParents() != null) && (!input.getParents().isEmpty())) {
158 partitionInfo.put("partitionParents", input.getParents().toString());
159 }
160 inputPartitionInfo.add(partitionInfo);
161 break;
162 default:
163 break;
164 }
165 }
166
167 outJSONObject.put("input_tables", inputTableInfo);
168 outJSONObject.put("input_partitions", inputPartitionInfo);
169 return outJSONObject;
170 }
171
172 public JSONObject getJSONLogicalPlan(PrintStream out, ExplainWork work) throws Exception {
173 isLogical = true;
174
175 JSONObject outJSONObject = new JSONObject(new LinkedHashMap<>());
176 boolean jsonOutput = work.isFormatted();
177 if (jsonOutput) {
178 out = null;
179 }
180
181 if (work.getParseContext() != null) {
182 if (out != null) {
183 out.print("LOGICAL PLAN:");
184 }
185 JSONObject jsonPlan = outputMap(work.getParseContext().getTopOps(), true,
186 out, work.getExtended(), jsonOutput, 0);
187 if (out != null) {
188 out.println();
189 }
190
191 if (jsonOutput) {
192 outJSONObject.put("LOGICAL PLAN", jsonPlan);
193 }
194 } else {
195 System.err.println("No parse context!");
196 }
197 return outJSONObject;
198 }
199
200 private static String trueCondNameVectorizationEnabled =
201 HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED.varname + " IS true";
202 private static String falseCondNameVectorizationEnabled =
203 HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED.varname + " IS false";
204
205 private ImmutablePair<Boolean, JSONObject> outputPlanVectorization(PrintStream out, boolean jsonOutput)
206 throws Exception {
207
208 if (out != null) {
209 out.println("PLAN VECTORIZATION:");
210 }
211
212 JSONObject json = jsonOutput ? new JSONObject(new LinkedHashMap<>()) : null;
213
214 HiveConf hiveConf = queryState.getConf();
215
216 boolean isVectorizationEnabled = HiveConf.getBoolVar(hiveConf,
217 HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED);
218 String isVectorizationEnabledCondName =
219 (isVectorizationEnabled ?
220 trueCondNameVectorizationEnabled :
221 falseCondNameVectorizationEnabled);
222 List<String> isVectorizationEnabledCondList = Arrays.asList(isVectorizationEnabledCondName);
223
224 if (out != null) {
225 out.print(indentString(2));
226 out.print("enabled: ");
227 out.println(isVectorizationEnabled);
228 out.print(indentString(2));
229 if (!isVectorizationEnabled) {
230 out.print("enabledConditionsNotMet: ");
231 } else {
232 out.print("enabledConditionsMet: ");
233 }
234 out.println(isVectorizationEnabledCondList);
235 }
236 if (jsonOutput) {
237 json.put("enabled", isVectorizationEnabled);
238 if (!isVectorizationEnabled) {
239 json.put("enabledConditionsNotMet", isVectorizationEnabledCondList);
240 } else {
241 json.put("enabledConditionsMet", isVectorizationEnabledCondList);
242 }
243 }
244
245 return new ImmutablePair<Boolean, JSONObject>(isVectorizationEnabled, jsonOutput ? json : null);
246 }
247
248 public JSONObject getJSONPlan(PrintStream out, ExplainWork work)
249 throws Exception {
250 return getJSONPlan(out, work.getRootTasks(), work.getFetchTask(),
251 work.isFormatted(), work.getExtended(), work.isAppendTaskType());
252 }
253
254 public JSONObject getJSONPlan(PrintStream out, List<Task<?>> tasks, Task<?> fetchTask,
255 boolean jsonOutput, boolean isExtended, boolean appendTaskType) throws Exception {
256
257 // If the user asked for a formatted output, dump the json output
258 // in the output stream
259 JSONObject outJSONObject = new JSONObject(new LinkedHashMap<>());
260
261 if (jsonOutput) {
262 out = null;
263 }
264
265 List<Task> ordered = StageIDsRearranger.getExplainOrder(conf, tasks);
266
267 if (fetchTask != null) {
268 fetchTask.setParentTasks((List)StageIDsRearranger.getFetchSources(tasks));
269 if (fetchTask.getNumParent() == 0) {
270 fetchTask.setRootTask(true);
271 }
272 ordered.add(fetchTask);
273 }
274
275 boolean suppressOthersForVectorization = false;
276 if (this.work != null && this.work.isVectorization()) {
277 ImmutablePair<Boolean, JSONObject> planVecPair = outputPlanVectorization(out, jsonOutput);
278
279 if (this.work.isVectorizationOnly()) {
280 // Suppress the STAGES if vectorization is off.
281 suppressOthersForVectorization = !planVecPair.left;
282 }
283
284 if (out != null) {
285 out.println();
286 }
287
288 if (jsonOutput) {
289 outJSONObject.put("PLAN VECTORIZATION", planVecPair.right);
290 }
291 }
292
293 if (!suppressOthersForVectorization) {
294 JSONObject jsonDependencies = outputDependencies(out, jsonOutput, appendTaskType, ordered);
295
296 if (out != null) {
297 out.println();
298 }
299
300 if (jsonOutput) {
301 outJSONObject.put("STAGE DEPENDENCIES", jsonDependencies);
302 }
303
304 // Go over all the tasks and dump out the plans
305 JSONObject jsonPlan = outputStagePlans(out, ordered,
306 jsonOutput, isExtended);
307
308 if (jsonOutput) {
309 outJSONObject.put("STAGE PLANS", jsonPlan);
310 }
311
312 if (fetchTask != null) {
313 fetchTask.setParentTasks(null);
314 }
315 }
316
317 return jsonOutput ? outJSONObject : null;
318 }
319
320 private List<String> toString(Collection<?> objects) {
321 List<String> list = new ArrayList<String>();
322 for (Object object : objects) {
323 list.add(String.valueOf(object));
324 }
325 return list;
326 }
327
328 private Object toJson(String header, String message, PrintStream out, ExplainWork work)
329 throws Exception {
330 if (work.isFormatted()) {
331 return message;
332 }
333 out.print(header);
334 out.println(": ");
335 out.print(indentString(2));
336 out.println(message);
337 return null;
338 }
339
340 private Object toJson(String header, List<String> messages, PrintStream out, ExplainWork work)
341 throws Exception {
342 if (work.isFormatted()) {
343 return new JSONArray(messages);
344 }
345 out.print(header);
346 out.println(": ");
347 for (String message : messages) {
348 out.print(indentString(2));
349 out.print(message);
350 out.println();
351 }
352 return null;
353 }
354
355 @Override
356 public int execute(DriverContext driverContext) {
357
358 PrintStream out = null;
359 try {
360 Path resFile = work.getResFile();
361 OutputStream outS = resFile.getFileSystem(conf).create(resFile);
362 out = new PrintStream(outS);
363
364 if (work.isLogical()) {
365 JSONObject jsonLogicalPlan = getJSONLogicalPlan(out, work);
366 if (work.isFormatted()) {
367 out.print(jsonLogicalPlan);
368 }
369 } else if (work.isAuthorize()) {
370 JSONObject jsonAuth = collectAuthRelatedEntities(out, work);
371 if (work.isFormatted()) {
372 out.print(jsonAuth);
373 }
374 } else if (work.getDependency()) {
375 JSONObject jsonDependencies = getJSONDependencies(work);
376 out.print(jsonDependencies);
377 } else {
378 if (work.isUserLevelExplain()) {
379 // Because of the implementation of the JsonParserFactory, we are sure
380 // that we can get a TezJsonParser.
381 JsonParser jsonParser = JsonParserFactory.getParser(conf);
382 work.getConfig().setFormatted(true);
383 JSONObject jsonPlan = getJSONPlan(out, work);
384 if (work.getCboInfo() != null) {
385 jsonPlan.put("cboInfo", work.getCboInfo());
386 }
387 try {
388 jsonParser.print(jsonPlan, out);
389 } catch (Exception e) {
390 // if there is anything wrong happen, we bail out.
391 LOG.error("Running explain user level has problem: " + e.toString()
392 + ". Falling back to normal explain");
393 work.getConfig().setFormatted(false);
394 work.getConfig().setUserLevelExplain(false);
395 jsonPlan = getJSONPlan(out, work);
396 }
397 } else {
398 JSONObject jsonPlan = getJSONPlan(out, work);
399 if (work.isFormatted()) {
400 out.print(jsonPlan);
401 }
402 }
403 }
404
405 out.close();
406 out = null;
407 return (0);
408 }
409 catch (Exception e) {
410 console.printError("Failed with exception " + e.getMessage(),
411 "\n" + StringUtils.stringifyException(e));
412 return (1);
413 }
414 finally {
415 IOUtils.closeStream(out);
416 }
417 }
418
419 private JSONObject collectAuthRelatedEntities(PrintStream out, ExplainWork work)
420 throws Exception {
421
422 BaseSemanticAnalyzer analyzer = work.getAnalyzer();
423 HiveOperation operation = queryState.getHiveOperation();
424
425 JSONObject object = new JSONObject(new LinkedHashMap<>());
426 Object jsonInput = toJson("INPUTS", toString(analyzer.getInputs()), out, work);
427 if (work.isFormatted()) {
428 object.put("INPUTS", jsonInput);
429 }
430 Object jsonOutput = toJson("OUTPUTS", toString(analyzer.getOutputs()), out, work);
431 if (work.isFormatted()) {
432 object.put("OUTPUTS", jsonOutput);
433 }
434 String userName = SessionState.get().getAuthenticator().getUserName();
435 Object jsonUser = toJson("CURRENT_USER", userName, out, work);
436 if (work.isFormatted()) {
437 object.put("CURRENT_USER", jsonUser);
438 }
439 Object jsonOperation = toJson("OPERATION", operation.name(), out, work);
440 if (work.isFormatted()) {
441 object.put("OPERATION", jsonOperation);
442 }
443 if (analyzer.skipAuthorization()) {
444 return object;
445 }
446
447 final List<String> exceptions = new ArrayList<String>();
448 Object delegate = SessionState.get().getActiveAuthorizer();
449 if (delegate != null) {
450 Class itface = SessionState.get().getAuthorizerInterface();
451 Object authorizer = AuthorizationFactory.create(delegate, itface,
452 new AuthorizationFactory.AuthorizationExceptionHandler() {
453 public void exception(Exception exception) {
454 exceptions.add(exception.getMessage());
455 }
456 });
457
458 SessionState.get().setActiveAuthorizer(authorizer);
459 try {
460 Driver.doAuthorization(queryState.getHiveOperation(), analyzer, "");
461 } finally {
462 SessionState.get().setActiveAuthorizer(delegate);
463 }
464 }
465 if (!exceptions.isEmpty()) {
466 Object jsonFails = toJson("AUTHORIZATION_FAILURES", exceptions, out, work);
467 if (work.isFormatted()) {
468 object.put("AUTHORIZATION_FAILURES", jsonFails);
469 }
470 }
471 return object;
472 }
473
474 private static String indentString(int indent) {
475 StringBuilder sb = new StringBuilder();
476 for (int i = 0; i < indent; ++i) {
477 sb.append(" ");
478 }
479
480 return sb.toString();
481 }
482
483 private JSONObject outputMap(Map<?, ?> mp, boolean hasHeader, PrintStream out,
484 boolean extended, boolean jsonOutput, int indent) throws Exception {
485
486 TreeMap<Object, Object> tree = getBasictypeKeyedMap(mp);
487 JSONObject json = jsonOutput ? new JSONObject(new LinkedHashMap<>()) : null;
488 if (out != null && hasHeader && !mp.isEmpty()) {
489 out.println();
490 }
491 for (Entry<?, ?> ent : tree.entrySet()) {
492 // Print the key
493 if (out != null) {
494 out.print(indentString(indent));
495 out.print(ent.getKey());
496 out.print(" ");
497 }
498
499 // Print the value
500 if (isPrintable(ent.getValue())) {
501 if (out != null) {
502 out.print(ent.getValue());
503 out.println();
504 }
505 if (jsonOutput) {
506 json.put(ent.getKey().toString(), ent.getValue().toString());
507 }
508 }
509 else if (ent.getValue() instanceof List) {
510 if (ent.getValue() != null && !((List<?>)ent.getValue()).isEmpty()
511 && ((List<?>)ent.getValue()).get(0) != null &&
512 ((List<?>)ent.getValue()).get(0) instanceof TezWork.Dependency) {
513 if (out != null) {
514 boolean isFirst = true;
515 for (TezWork.Dependency dep: (List<TezWork.Dependency>)ent.getValue()) {
516 if (!isFirst) {
517 out.print(", ");
518 } else {
519 out.print("<- ");
520 isFirst = false;
521 }
522 out.print(dep.getName());
523 out.print(" (");
524 out.print(dep.getType());
525 out.print(")");
526 }
527 out.println();
528 }
529 if (jsonOutput) {
530 for (TezWork.Dependency dep: (List<TezWork.Dependency>)ent.getValue()) {
531 JSONObject jsonDep = new JSONObject(new LinkedHashMap<>());
532 jsonDep.put("parent", dep.getName());
533 jsonDep.put("type", dep.getType());
534 json.accumulate(ent.getKey().toString(), jsonDep);
535 }
536 }
537 } else if (ent.getValue() != null && !((List<?>) ent.getValue()).isEmpty()
538 && ((List<?>) ent.getValue()).get(0) != null &&
539 ((List<?>) ent.getValue()).get(0) instanceof SparkWork.Dependency) {
540 if (out != null) {
541 boolean isFirst = true;
542 for (SparkWork.Dependency dep: (List<SparkWork.Dependency>) ent.getValue()) {
543 if (!isFirst) {
544 out.print(", ");
545 } else {
546 out.print("<- ");
547 isFirst = false;
548 }
549 out.print(dep.getName());
550 out.print(" (");
551 out.print(dep.getShuffleType());
552 out.print(", ");
553 out.print(dep.getNumPartitions());
554 out.print(")");
555 }
556 out.println();
557 }
558 if (jsonOutput) {
559 for (SparkWork.Dependency dep: (List<SparkWork.Dependency>) ent.getValue()) {
560 JSONObject jsonDep = new JSONObject(new LinkedHashMap<>());
561 jsonDep.put("parent", dep.getName());
562 jsonDep.put("type", dep.getShuffleType());
563 jsonDep.put("partitions", dep.getNumPartitions());
564 json.accumulate(ent.getKey().toString(), jsonDep);
565 }
566 }
567 } else {
568 if (out != null) {
569 out.print(ent.getValue().toString());
570 out.println();
571 }
572 if (jsonOutput) {
573 json.put(ent.getKey().toString(), ent.getValue().toString());
574 }
575 }
576 }
577 else if (ent.getValue() instanceof Map) {
578 String stringValue = getBasictypeKeyedMap((Map)ent.getValue()).toString();
579 if (out != null) {
580 out.print(stringValue);
581 out.println();
582 }
583 if (jsonOutput) {
584 json.put(ent.getKey().toString(), stringValue);
585 }
586 }
587 else if (ent.getValue() != null) {
588 if (out != null) {
589 out.println();
590 }
591 JSONObject jsonOut = outputPlan(ent.getValue(), out,
592 extended, jsonOutput, jsonOutput ? 0 : indent + 2);
593 if (jsonOutput) {
594 json.put(ent.getKey().toString(), jsonOut);
595 }
596 }
597 else {
598 if (out != null) {
599 out.println();
600 }
601 }
602 }
603
604 return jsonOutput ? json : null;
605 }
606
607 /**
608 * Retruns a map which have either primitive or string keys.
609 *
610 * This is neccessary to discard object level comparators which may sort the objects based on some non-trivial logic.
611 *
612 * @param mp
613 * @return
614 */
615 private TreeMap<Object, Object> getBasictypeKeyedMap(Map<?, ?> mp) {
616 TreeMap<Object, Object> ret = new TreeMap<Object, Object>();
617 if (mp.size() > 0) {
618 Object firstKey = mp.keySet().iterator().next();
619 if (firstKey.getClass().isPrimitive() || firstKey instanceof String) {
620 // keep it as-is
621 ret.putAll(mp);
622 return ret;
623 } else {
624 for (Entry<?, ?> entry : mp.entrySet()) {
625 // discard possibly type related sorting order and replace with alphabetical
626 ret.put(entry.getKey().toString(), entry.getValue());
627 }
628 }
629 }
630 return ret;
631 }
632
633 private JSONArray outputList(List<?> l, PrintStream out, boolean hasHeader,
634 boolean extended, boolean jsonOutput, int indent) throws Exception {
635
636 boolean first_el = true;
637 boolean nl = false;
638 JSONArray outputArray = new JSONArray();
639
640 for (Object o : l) {
641 if (isPrintable(o)) {
642 String delim = first_el ? " " : ", ";
643 if (out != null) {
644 out.print(delim);
645 out.print(o);
646 }
647
648 if (jsonOutput) {
649 outputArray.put(o);
650 }
651 nl = true;
652 }
653 else {
654 if (first_el && (out != null) && hasHeader) {
655 out.println();
656 }
657 JSONObject jsonOut = outputPlan(o, out, extended,
658 jsonOutput, jsonOutput ? 0 : (hasHeader ? indent + 2 : indent));
659 if (jsonOutput) {
660 outputArray.put(jsonOut);
661 }
662 }
663
664 first_el = false;
665 }
666
667 if (nl && (out != null)) {
668 out.println();
669 }
670
671 return jsonOutput ? outputArray : null;
672 }
673
674 private boolean isPrintable(Object val) {
675 if (val instanceof Boolean || val instanceof String
676 || val instanceof Integer || val instanceof Long || val instanceof Byte
677 || val instanceof Float || val instanceof Double || val instanceof Path) {
678 return true;
679 }
680
681 if (val != null && val.getClass().isPrimitive()) {
682 return true;
683 }
684
685 return false;
686 }
687
688 private JSONObject outputPlan(Object work,
689 PrintStream out, boolean extended, boolean jsonOutput, int indent) throws Exception {
690 return outputPlan(work, out, extended, jsonOutput, indent, "");
691 }
692
693 private JSONObject outputPlan(Object work, PrintStream out,
694 boolean extended, boolean jsonOutput, int indent, String appendToHeader) throws Exception {
695 // Check if work has an explain annotation
696 Annotation note = AnnotationUtils.getAnnotation(work.getClass(), Explain.class);
697
698 String keyJSONObject = null;
699
700 if (note instanceof Explain) {
701 Explain xpl_note = (Explain) note;
702 boolean invokeFlag = false;
703 if (this.work != null && this.work.isUserLevelExplain()) {
704 invokeFlag = Level.USER.in(xpl_note.explainLevels());
705 } else {
706 if (extended) {
707 invokeFlag = Level.EXTENDED.in(xpl_note.explainLevels());
708 } else {
709 invokeFlag = Level.DEFAULT.in(xpl_note.explainLevels());
710 }
711 }
712 if (invokeFlag) {
713 Vectorization vectorization = xpl_note.vectorization();
714 if (this.work != null && this.work.isVectorization()) {
715
716 // The EXPLAIN VECTORIZATION option was specified.
717 final boolean desireOnly = this.work.isVectorizationOnly();
718 final VectorizationDetailLevel desiredVecDetailLevel =
719 this.work.isVectorizationDetailLevel();
720
721 switch (vectorization) {
722 case NON_VECTORIZED:
723 // Display all non-vectorized leaf objects unless ONLY.
724 if (desireOnly) {
725 invokeFlag = false;
726 }
727 break;
728 case SUMMARY:
729 case OPERATOR:
730 case EXPRESSION:
731 case DETAIL:
732 if (vectorization.rank < desiredVecDetailLevel.rank) {
733 // This detail not desired.
734 invokeFlag = false;
735 }
736 break;
737 case SUMMARY_PATH:
738 case OPERATOR_PATH:
739 if (desireOnly) {
740 if (vectorization.rank < desiredVecDetailLevel.rank) {
741 // Suppress headers and all objects below.
742 invokeFlag = false;
743 }
744 }
745 break;
746 default:
747 throw new RuntimeException("Unknown EXPLAIN vectorization " + vectorization);
748 }
749 } else {
750 // Do not display vectorization objects.
751 switch (vectorization) {
752 case SUMMARY:
753 case OPERATOR:
754 case EXPRESSION:
755 case DETAIL:
756 invokeFlag = false;
757 break;
758 case NON_VECTORIZED:
759 // No action.
760 break;
761 case SUMMARY_PATH:
762 case OPERATOR_PATH:
763 // Always include headers since they contain non-vectorized objects, too.
764 break;
765 default:
766 throw new RuntimeException("Unknown EXPLAIN vectorization " + vectorization);
767 }
768 }
769 }
770 if (invokeFlag) {
771 keyJSONObject = xpl_note.displayName();
772 if (out != null) {
773 out.print(indentString(indent));
774 if (appendToHeader != null && !appendToHeader.isEmpty()) {
775 out.println(xpl_note.displayName() + appendToHeader);
776 } else {
777 out.println(xpl_note.displayName());
778 }
779 }
780 }
781 }
782
783 JSONObject json = jsonOutput ? new JSONObject(new LinkedHashMap<>()) : null;
784 // If this is an operator then we need to call the plan generation on the
785 // conf and then the children
786 if (work instanceof Operator) {
787 Operator<? extends OperatorDesc> operator =
788 (Operator<? extends OperatorDesc>) work;
789 if (operator.getConf() != null) {
790 String appender = isLogical ? " (" + operator.getOperatorId() + ")" : "";
791 JSONObject jsonOut = outputPlan(operator.getConf(), out, extended,
792 jsonOutput, jsonOutput ? 0 : indent, appender);
793 if (this.work != null && this.work.isUserLevelExplain()) {
794 if (jsonOut != null && jsonOut.length() > 0) {
795 ((JSONObject) jsonOut.get(JSONObject.getNames(jsonOut)[0])).put("OperatorId:",
796 operator.getOperatorId());
797 }
798 }
799 if (jsonOutput) {
800 json = jsonOut;
801 }
802 }
803
804 if (!visitedOps.contains(operator) || !isLogical) {
805 visitedOps.add(operator);
806 if (operator.getChildOperators() != null) {
807 int cindent = jsonOutput ? 0 : indent + 2;
808 for (Operator<? extends OperatorDesc> op : operator.getChildOperators()) {
809 JSONObject jsonOut = outputPlan(op, out, extended, jsonOutput, cindent);
810 if (jsonOutput) {
811 ((JSONObject)json.get(JSONObject.getNames(json)[0])).accumulate("children", jsonOut);
812 }
813 }
814 }
815 }
816
817 if (jsonOutput) {
818 return json;
819 }
820 return null;
821 }
822
823 // We look at all methods that generate values for explain
824 Method[] methods = work.getClass().getMethods();
825 Arrays.sort(methods, new MethodComparator());
826
827 for (Method m : methods) {
828 int prop_indents = jsonOutput ? 0 : indent + 2;
829 note = AnnotationUtils.getAnnotation(m, Explain.class);
830
831 if (note instanceof Explain) {
832 Explain xpl_note = (Explain) note;
833 boolean invokeFlag = false;
834 if (this.work != null && this.work.isUserLevelExplain()) {
835 invokeFlag = Level.USER.in(xpl_note.explainLevels());
836 } else {
837 if (extended) {
838 invokeFlag = Level.EXTENDED.in(xpl_note.explainLevels());
839 } else {
840 invokeFlag = Level.DEFAULT.in(xpl_note.explainLevels());
841 }
842 }
843 if (invokeFlag) {
844 Vectorization vectorization = xpl_note.vectorization();
845 if (this.work != null && this.work.isVectorization()) {
846
847 // The EXPLAIN VECTORIZATION option was specified.
848 final boolean desireOnly = this.work.isVectorizationOnly();
849 final VectorizationDetailLevel desiredVecDetailLevel =
850 this.work.isVectorizationDetailLevel();
851
852 switch (vectorization) {
853 case NON_VECTORIZED:
854 // Display all non-vectorized leaf objects unless ONLY.
855 if (desireOnly) {
856 invokeFlag = false;
857 }
858 break;
859 case SUMMARY:
860 case OPERATOR:
861 case EXPRESSION:
862 case DETAIL:
863 if (vectorization.rank < desiredVecDetailLevel.rank) {
864 // This detail not desired.
865 invokeFlag = false;
866 }
867 break;
868 case SUMMARY_PATH:
869 case OPERATOR_PATH:
870 if (desireOnly) {
871 if (vectorization.rank < desiredVecDetailLevel.rank) {
872 // Suppress headers and all objects below.
873 invokeFlag = false;
874 }
875 }
876 break;
877 default:
878 throw new RuntimeException("Unknown EXPLAIN vectorization " + vectorization);
879 }
880 } else {
881 // Do not display vectorization objects.
882 switch (vectorization) {
883 case SUMMARY:
884 case OPERATOR:
885 case EXPRESSION:
886 case DETAIL:
887 invokeFlag = false;
888 break;
889 case NON_VECTORIZED:
890 // No action.
891 break;
892 case SUMMARY_PATH:
893 case OPERATOR_PATH:
894 // Always include headers since they contain non-vectorized objects, too.
895 break;
896 default:
897 throw new RuntimeException("Unknown EXPLAIN vectorization " + vectorization);
898 }
899 }
900 }
901 if (invokeFlag) {
902
903 Object val = null;
904 try {
905 val = m.invoke(work);
906 }
907 catch (InvocationTargetException ex) {
908 // Ignore the exception, this may be caused by external jars
909 val = null;
910 }
911
912 if (val == null) {
913 continue;
914 }
915
916 String header = null;
917 boolean skipHeader = xpl_note.skipHeader();
918 boolean emptyHeader = false;
919
920 if (!xpl_note.displayName().equals("")) {
921 header = indentString(prop_indents) + xpl_note.displayName() + ":";
922 }
923 else {
924 emptyHeader = true;
925 prop_indents = indent;
926 header = indentString(prop_indents);
927 }
928
929 // Try the output as a primitive object
930 if (isPrintable(val)) {
931 if (out != null && shouldPrint(xpl_note, val)) {
932 if (!skipHeader) {
933 out.print(header);
934 out.print(" ");
935 }
936 out.println(val);
937 }
938 if (jsonOutput && shouldPrint(xpl_note, val)) {
939 json.put(header, val.toString());
940 }
941 continue;
942 }
943
944 int ind = 0;
945 if (!jsonOutput) {
946 if (!skipHeader) {
947 ind = prop_indents + 2;
948 } else {
949 ind = indent;
950 }
951 }
952
953 // Try this as a map
954 if (val instanceof Map) {
955 // Go through the map and print out the stuff
956 Map<?, ?> mp = (Map<?, ?>) val;
957
958 if (out != null && !skipHeader && mp != null && !mp.isEmpty()) {
959 out.print(header);
960 }
961
962 JSONObject jsonOut = outputMap(mp, !skipHeader && !emptyHeader, out, extended, jsonOutput, ind);
963 if (jsonOutput && !mp.isEmpty()) {
964 json.put(header, jsonOut);
965 }
966 continue;
967 }
968
969 // Try this as a list
970 if (val instanceof List || val instanceof Set) {
971 List l = val instanceof List ? (List)val : new ArrayList((Set)val);
972 if (out != null && !skipHeader && l != null && !l.isEmpty()) {
973 out.print(header);
974 }
975
976 JSONArray jsonOut = outputList(l, out, !skipHeader && !emptyHeader, extended, jsonOutput, ind);
977
978 if (jsonOutput && !l.isEmpty()) {
979 json.put(header, jsonOut);
980 }
981
982 continue;
983 }
984
985 // Finally check if it is serializable
986 try {
987 if (!skipHeader && out != null) {
988 out.println(header);
989 }
990 JSONObject jsonOut = outputPlan(val, out, extended, jsonOutput, ind);
991 if (jsonOutput && jsonOut != null && jsonOut.length() != 0) {
992 if (!skipHeader) {
993 json.put(header, jsonOut);
994 } else {
995 for(String k: JSONObject.getNames(jsonOut)) {
996 json.put(k, jsonOut.get(k));
997 }
998 }
999 }
1000 continue;
1001 }
1002 catch (ClassCastException ce) {
1003 // Ignore
1004 }
1005 }
1006 }
1007 }
1008
1009 if (jsonOutput) {
1010 if (keyJSONObject != null) {
1011 JSONObject ret = new JSONObject(new LinkedHashMap<>());
1012 ret.put(keyJSONObject, json);
1013 return ret;
1014 }
1015
1016 return json;
1017 }
1018 return null;
1019 }
1020
1021 /**
1022 * use case: we want to print the object in explain only if it is true
1023 * how to do : print it unless the following 3 are all true:
1024 * 1. displayOnlyOnTrue tag is on
1025 * 2. object is boolean
1026 * 3. object is false
1027 * @param exp
1028 * @param val
1029 * @return
1030 */
1031 private boolean shouldPrint(Explain exp, Object val) {
1032 if (exp.displayOnlyOnTrue() && (val instanceof Boolean) & !((Boolean)val)) {
1033 return false;
1034 }
1035 return true;
1036 }
1037
1038 private JSONObject outputPlan(Task<?> task,
1039 PrintStream out, JSONObject parentJSON, boolean extended,
1040 boolean jsonOutput, int indent) throws Exception {
1041
1042 if (out != null) {
1043 out.print(indentString(indent));
1044 out.print("Stage: ");
1045 out.print(task.getId());
1046 out.print("\n");
1047 }
1048
1049 // Start by getting the work part of the task and call the output plan for
1050 // the work
1051 JSONObject jsonOutputPlan = outputPlan(task.getWork(), out, extended,
1052 jsonOutput, jsonOutput ? 0 : indent + 2);
1053
1054 if (out != null) {
1055 out.println();
1056 }
1057
1058 if (jsonOutput) {
1059 parentJSON.put(task.getId(), jsonOutputPlan);
1060 }
1061 return null;
1062 }
1063
1064 private JSONObject outputDependencies(Task<?> task,
1065 PrintStream out, JSONObject parentJson, boolean jsonOutput, boolean taskType, int indent)
1066 throws Exception {
1067
1068 boolean first = true;
1069 JSONObject json = jsonOutput ? new JSONObject(new LinkedHashMap<>()) : null;
1070 if (out != null) {
1071 out.print(indentString(indent));
1072 out.print(task.getId());
1073 }
1074
1075 if ((task.getParentTasks() == null || task.getParentTasks().isEmpty())) {
1076 if (task.isRootTask()) {
1077 if (out != null) {
1078 out.print(" is a root stage");
1079 }
1080
1081 if (jsonOutput) {
1082 json.put("ROOT STAGE", "TRUE");
1083 }
1084 }
1085 }
1086 else {
1087 StringBuilder s = new StringBuilder();
1088 first = true;
1089 for (Task<?> parent : task.getParentTasks()) {
1090 if (!first) {
1091 s.append(", ");
1092 }
1093 first = false;
1094 s.append(parent.getId());
1095 }
1096
1097 if (out != null) {
1098 out.print(" depends on stages: ");
1099 out.print(s.toString());
1100 }
1101 if (jsonOutput) {
1102 json.put("DEPENDENT STAGES", s.toString());
1103 }
1104 }
1105
1106 Task<?> currBackupTask = task.getBackupTask();
1107 if (currBackupTask != null) {
1108 if (out != null) {
1109 out.print(" has a backup stage: ");
1110 out.print(currBackupTask.getId());
1111 }
1112 if (jsonOutput) {
1113 json.put("BACKUP STAGE", currBackupTask.getId());
1114 }
1115 }
1116
1117 if (task instanceof ConditionalTask
1118 && ((ConditionalTask) task).getListTasks() != null) {
1119 StringBuilder s = new StringBuilder();
1120 first = true;
1121 for (Task<?> con : ((ConditionalTask) task).getListTasks()) {
1122 if (!first) {
1123 s.append(", ");
1124 }
1125 first = false;
1126 s.append(con.getId());
1127 }
1128
1129 if (out != null) {
1130 out.print(" , consists of ");
1131 out.print(s.toString());
1132 }
1133 if (jsonOutput) {
1134 json.put("CONDITIONAL CHILD TASKS", s.toString());
1135 }
1136 }
1137 if (taskType) {
1138 if (out != null) {
1139 out.print(" [");
1140 out.print(task.getType());
1141 out.print("]");
1142 }
1143 if (jsonOutput) {
1144 json.put("TASK TYPE", task.getType().name());
1145 }
1146 }
1147
1148 if (out != null) {
1149 out.println();
1150 }
1151 return jsonOutput ? json : null;
1152 }
1153
1154 public String outputAST(String treeString, PrintStream out,
1155 boolean jsonOutput, int indent) throws JSONException {
1156 if (out != null) {
1157 out.print(indentString(indent));
1158 out.println("ABSTRACT SYNTAX TREE:");
1159 out.print(indentString(indent + 2));
1160 out.println(treeString);
1161 }
1162
1163 return jsonOutput ? treeString : null;
1164 }
1165
1166 public JSONObject outputDependencies(PrintStream out, boolean jsonOutput,
1167 boolean appendTaskType, List<Task> tasks)
1168 throws Exception {
1169
1170 if (out != null) {
1171 out.println("STAGE DEPENDENCIES:");
1172 }
1173
1174 JSONObject json = jsonOutput ? new JSONObject(new LinkedHashMap<>()) : null;
1175 for (Task task : tasks) {
1176 JSONObject jsonOut = outputDependencies(task, out, json, jsonOutput, appendTaskType, 2);
1177 if (jsonOutput && jsonOut != null) {
1178 json.put(task.getId(), jsonOut);
1179 }
1180 }
1181
1182 return jsonOutput ? json : null;
1183 }
1184
1185 public JSONObject outputStagePlans(PrintStream out, List<Task> tasks,
1186 boolean jsonOutput, boolean isExtended)
1187 throws Exception {
1188
1189 if (out != null) {
1190 out.println("STAGE PLANS:");
1191 }
1192
1193 JSONObject json = jsonOutput ? new JSONObject(new LinkedHashMap<>()) : null;
1194 for (Task task : tasks) {
1195 outputPlan(task, out, json, isExtended, jsonOutput, 2);
1196 }
1197 return jsonOutput ? json : null;
1198 }
1199
1200 /**
1201 * MethodComparator.
1202 *
1203 */
1204 public class MethodComparator implements Comparator<Method> {
1205 @Override
1206 public int compare(Method m1, Method m2) {
1207 return m1.getName().compareTo(m2.getName());
1208 }
1209 }
1210
1211 @Override
1212 public StageType getType() {
1213 return StageType.EXPLAIN;
1214 }
1215
1216 @Override
1217 public String getName() {
1218 return "EXPLAIN";
1219 }
1220
1221 public List<FieldSchema> getResultSchema() {
1222 FieldSchema tmpFieldSchema = new FieldSchema();
1223 List<FieldSchema> colList = new ArrayList<FieldSchema>();
1224
1225 tmpFieldSchema.setName(EXPL_COLUMN_NAME);
1226 tmpFieldSchema.setType(STRING_TYPE_NAME);
1227
1228 colList.add(tmpFieldSchema);
1229 return colList;
1230 }
1231 }