b8a4693a09baa069e812436f600c31cbb9300d82
[hive.git] / ql / src / java / org / apache / hadoop / hive / ql / exec / ExplainTask.java
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 package org.apache.hadoop.hive.ql.exec;
20
21 import static org.apache.hadoop.hive.serde.serdeConstants.STRING_TYPE_NAME;
22
23 import java.io.OutputStream;
24 import java.io.PrintStream;
25 import java.io.Serializable;
26 import java.lang.annotation.Annotation;
27 import java.lang.reflect.InvocationTargetException;
28 import java.lang.reflect.Method;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.Collection;
32 import java.util.Comparator;
33 import java.util.HashMap;
34 import java.util.HashSet;
35 import java.util.LinkedHashMap;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Map.Entry;
39 import java.util.Set;
40 import java.util.TreeMap;
41
42 import org.apache.hadoop.fs.Path;
43 import org.apache.hadoop.hive.common.jsonexplain.JsonParser;
44 import org.apache.hadoop.hive.common.jsonexplain.JsonParserFactory;
45 import org.apache.hadoop.hive.metastore.api.FieldSchema;
46 import org.apache.hadoop.hive.ql.Driver;
47 import org.apache.hadoop.hive.ql.DriverContext;
48 import org.apache.hadoop.hive.ql.hooks.ReadEntity;
49 import org.apache.hadoop.hive.ql.metadata.Table;
50 import org.apache.hadoop.hive.ql.optimizer.physical.StageIDsRearranger;
51 import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
52 import org.apache.hadoop.hive.ql.plan.Explain;
53 import org.apache.hadoop.hive.ql.plan.Explain.Level;
54 import org.apache.hadoop.hive.ql.plan.ExplainWork;
55 import org.apache.hadoop.hive.ql.plan.HiveOperation;
56 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
57 import org.apache.hadoop.hive.ql.plan.SparkWork;
58 import org.apache.hadoop.hive.ql.plan.TezWork;
59 import org.apache.hadoop.hive.ql.plan.api.StageType;
60 import org.apache.hadoop.hive.ql.security.authorization.AuthorizationFactory;
61 import org.apache.hadoop.hive.ql.session.SessionState;
62 import org.apache.hadoop.io.IOUtils;
63 import org.apache.hadoop.util.StringUtils;
64 import org.apache.hive.common.util.AnnotationUtils;
65 import org.json.JSONArray;
66 import org.json.JSONException;
67 import org.json.JSONObject;
68 import org.slf4j.Logger;
69 import org.slf4j.LoggerFactory;
70
71 /**
72 * ExplainTask implementation.
73 *
74 **/
75 public class ExplainTask extends Task<ExplainWork> implements Serializable {
76 private static final long serialVersionUID = 1L;
77 public static final String EXPL_COLUMN_NAME = "Explain";
78 private final Set<Operator<?>> visitedOps = new HashSet<Operator<?>>();
79 private boolean isLogical = false;
80 protected final Logger LOG;
81
82 public ExplainTask() {
83 super();
84 LOG = LoggerFactory.getLogger(this.getClass().getName());
85 }
86
87 /*
88 * Below method returns the dependencies for the passed in query to EXPLAIN.
89 * The dependencies are the set of input tables and partitions, and are
90 * provided back as JSON output for the EXPLAIN command.
91 * Example output:
92 * {"input_tables":[{"tablename": "default@test_sambavi_v1", "tabletype": "TABLE"}],
93 * "input partitions":["default@srcpart@ds=2008-04-08/hr=11"]}
94 */
95 private static JSONObject getJSONDependencies(ExplainWork work)
96 throws Exception {
97 assert(work.getDependency());
98
99 JSONObject outJSONObject = new JSONObject(new LinkedHashMap<>());
100 List<Map<String, String>> inputTableInfo = new ArrayList<Map<String, String>>();
101 List<Map<String, String>> inputPartitionInfo = new ArrayList<Map<String, String>>();
102 for (ReadEntity input: work.getInputs()) {
103 switch (input.getType()) {
104 case TABLE:
105 Table table = input.getTable();
106 Map<String, String> tableInfo = new LinkedHashMap<String, String>();
107 tableInfo.put("tablename", table.getCompleteName());
108 tableInfo.put("tabletype", table.getTableType().toString());
109 if ((input.getParents() != null) && (!input.getParents().isEmpty())) {
110 tableInfo.put("tableParents", input.getParents().toString());
111 }
112 inputTableInfo.add(tableInfo);
113 break;
114 case PARTITION:
115 Map<String, String> partitionInfo = new HashMap<String, String>();
116 partitionInfo.put("partitionName", input.getPartition().getCompleteName());
117 if ((input.getParents() != null) && (!input.getParents().isEmpty())) {
118 partitionInfo.put("partitionParents", input.getParents().toString());
119 }
120 inputPartitionInfo.add(partitionInfo);
121 break;
122 default:
123 break;
124 }
125 }
126
127 outJSONObject.put("input_tables", inputTableInfo);
128 outJSONObject.put("input_partitions", inputPartitionInfo);
129 return outJSONObject;
130 }
131
132 public JSONObject getJSONLogicalPlan(PrintStream out, ExplainWork work) throws Exception {
133 isLogical = true;
134
135 JSONObject outJSONObject = new JSONObject(new LinkedHashMap<>());
136 boolean jsonOutput = work.isFormatted();
137 if (jsonOutput) {
138 out = null;
139 }
140
141 if (work.getParseContext() != null) {
142 if (out != null) {
143 out.print("LOGICAL PLAN:");
144 }
145 JSONObject jsonPlan = outputMap(work.getParseContext().getTopOps(), true,
146 out, work.getExtended(), jsonOutput, 0);
147 if (out != null) {
148 out.println();
149 }
150
151 if (jsonOutput) {
152 outJSONObject.put("LOGICAL PLAN", jsonPlan);
153 }
154 } else {
155 System.err.println("No parse context!");
156 }
157 return outJSONObject;
158 }
159
160 public JSONObject getJSONPlan(PrintStream out, ExplainWork work)
161 throws Exception {
162 return getJSONPlan(out, work.getRootTasks(), work.getFetchTask(),
163 work.isFormatted(), work.getExtended(), work.isAppendTaskType());
164 }
165
166 public JSONObject getJSONPlan(PrintStream out, List<Task<?>> tasks, Task<?> fetchTask,
167 boolean jsonOutput, boolean isExtended, boolean appendTaskType) throws Exception {
168
169 // If the user asked for a formatted output, dump the json output
170 // in the output stream
171 JSONObject outJSONObject = new JSONObject(new LinkedHashMap<>());
172
173 if (jsonOutput) {
174 out = null;
175 }
176
177 List<Task> ordered = StageIDsRearranger.getExplainOrder(conf, tasks);
178
179 if (fetchTask != null) {
180 fetchTask.setParentTasks((List)StageIDsRearranger.getFetchSources(tasks));
181 if (fetchTask.getNumParent() == 0) {
182 fetchTask.setRootTask(true);
183 }
184 ordered.add(fetchTask);
185 }
186
187 JSONObject jsonDependencies = outputDependencies(out, jsonOutput, appendTaskType, ordered);
188
189 if (out != null) {
190 out.println();
191 }
192
193 if (jsonOutput) {
194 outJSONObject.put("STAGE DEPENDENCIES", jsonDependencies);
195 }
196
197 // Go over all the tasks and dump out the plans
198 JSONObject jsonPlan = outputStagePlans(out, ordered,
199 jsonOutput, isExtended);
200
201 if (jsonOutput) {
202 outJSONObject.put("STAGE PLANS", jsonPlan);
203 }
204
205 if (fetchTask != null) {
206 fetchTask.setParentTasks(null);
207 }
208
209 return jsonOutput ? outJSONObject : null;
210 }
211
212 private List<String> toString(Collection<?> objects) {
213 List<String> list = new ArrayList<String>();
214 for (Object object : objects) {
215 list.add(String.valueOf(object));
216 }
217 return list;
218 }
219
220 private Object toJson(String header, String message, PrintStream out, ExplainWork work)
221 throws Exception {
222 if (work.isFormatted()) {
223 return message;
224 }
225 out.print(header);
226 out.println(": ");
227 out.print(indentString(2));
228 out.println(message);
229 return null;
230 }
231
232 private Object toJson(String header, List<String> messages, PrintStream out, ExplainWork work)
233 throws Exception {
234 if (work.isFormatted()) {
235 return new JSONArray(messages);
236 }
237 out.print(header);
238 out.println(": ");
239 for (String message : messages) {
240 out.print(indentString(2));
241 out.print(message);
242 out.println();
243 }
244 return null;
245 }
246
247 @Override
248 public int execute(DriverContext driverContext) {
249
250 PrintStream out = null;
251 try {
252 Path resFile = work.getResFile();
253 OutputStream outS = resFile.getFileSystem(conf).create(resFile);
254 out = new PrintStream(outS);
255
256 if (work.isLogical()) {
257 JSONObject jsonLogicalPlan = getJSONLogicalPlan(out, work);
258 if (work.isFormatted()) {
259 out.print(jsonLogicalPlan);
260 }
261 } else if (work.isAuthorize()) {
262 JSONObject jsonAuth = collectAuthRelatedEntities(out, work);
263 if (work.isFormatted()) {
264 out.print(jsonAuth);
265 }
266 } else if (work.getDependency()) {
267 JSONObject jsonDependencies = getJSONDependencies(work);
268 out.print(jsonDependencies);
269 } else {
270 if (work.isUserLevelExplain()) {
271 // Because of the implementation of the JsonParserFactory, we are sure
272 // that we can get a TezJsonParser.
273 JsonParser jsonParser = JsonParserFactory.getParser(conf);
274 work.getConfig().setFormatted(true);
275 JSONObject jsonPlan = getJSONPlan(out, work);
276 if (work.getCboInfo() != null) {
277 jsonPlan.put("cboInfo", work.getCboInfo());
278 }
279 try {
280 jsonParser.print(jsonPlan, out);
281 } catch (Exception e) {
282 // if there is anything wrong happen, we bail out.
283 LOG.error("Running explain user level has problem: " + e.toString()
284 + ". Falling back to normal explain");
285 work.getConfig().setFormatted(false);
286 work.getConfig().setUserLevelExplain(false);
287 jsonPlan = getJSONPlan(out, work);
288 }
289 } else {
290 JSONObject jsonPlan = getJSONPlan(out, work);
291 if (work.isFormatted()) {
292 out.print(jsonPlan);
293 }
294 }
295 }
296
297 out.close();
298 out = null;
299 return (0);
300 }
301 catch (Exception e) {
302 console.printError("Failed with exception " + e.getMessage(),
303 "\n" + StringUtils.stringifyException(e));
304 return (1);
305 }
306 finally {
307 IOUtils.closeStream(out);
308 }
309 }
310
311 private JSONObject collectAuthRelatedEntities(PrintStream out, ExplainWork work)
312 throws Exception {
313
314 BaseSemanticAnalyzer analyzer = work.getAnalyzer();
315 HiveOperation operation = queryState.getHiveOperation();
316
317 JSONObject object = new JSONObject(new LinkedHashMap<>());
318 Object jsonInput = toJson("INPUTS", toString(analyzer.getInputs()), out, work);
319 if (work.isFormatted()) {
320 object.put("INPUTS", jsonInput);
321 }
322 Object jsonOutput = toJson("OUTPUTS", toString(analyzer.getOutputs()), out, work);
323 if (work.isFormatted()) {
324 object.put("OUTPUTS", jsonOutput);
325 }
326 String userName = SessionState.get().getAuthenticator().getUserName();
327 Object jsonUser = toJson("CURRENT_USER", userName, out, work);
328 if (work.isFormatted()) {
329 object.put("CURRENT_USER", jsonUser);
330 }
331 Object jsonOperation = toJson("OPERATION", operation.name(), out, work);
332 if (work.isFormatted()) {
333 object.put("OPERATION", jsonOperation);
334 }
335 if (analyzer.skipAuthorization()) {
336 return object;
337 }
338
339 final List<String> exceptions = new ArrayList<String>();
340 Object delegate = SessionState.get().getActiveAuthorizer();
341 if (delegate != null) {
342 Class itface = SessionState.get().getAuthorizerInterface();
343 Object authorizer = AuthorizationFactory.create(delegate, itface,
344 new AuthorizationFactory.AuthorizationExceptionHandler() {
345 public void exception(Exception exception) {
346 exceptions.add(exception.getMessage());
347 }
348 });
349
350 SessionState.get().setActiveAuthorizer(authorizer);
351 try {
352 Driver.doAuthorization(queryState.getHiveOperation(), analyzer, "");
353 } finally {
354 SessionState.get().setActiveAuthorizer(delegate);
355 }
356 }
357 if (!exceptions.isEmpty()) {
358 Object jsonFails = toJson("AUTHORIZATION_FAILURES", exceptions, out, work);
359 if (work.isFormatted()) {
360 object.put("AUTHORIZATION_FAILURES", jsonFails);
361 }
362 }
363 return object;
364 }
365
366 private static String indentString(int indent) {
367 StringBuilder sb = new StringBuilder();
368 for (int i = 0; i < indent; ++i) {
369 sb.append(" ");
370 }
371
372 return sb.toString();
373 }
374
375 private JSONObject outputMap(Map<?, ?> mp, boolean hasHeader, PrintStream out,
376 boolean extended, boolean jsonOutput, int indent) throws Exception {
377
378 TreeMap<Object, Object> tree = getBasictypeKeyedMap(mp);
379 JSONObject json = jsonOutput ? new JSONObject(new LinkedHashMap<>()) : null;
380 if (out != null && hasHeader && !mp.isEmpty()) {
381 out.println();
382 }
383 for (Entry<?, ?> ent : tree.entrySet()) {
384 // Print the key
385 if (out != null) {
386 out.print(indentString(indent));
387 out.print(ent.getKey());
388 out.print(" ");
389 }
390
391 // Print the value
392 if (isPrintable(ent.getValue())) {
393 if (out != null) {
394 out.print(ent.getValue());
395 out.println();
396 }
397 if (jsonOutput) {
398 json.put(ent.getKey().toString(), ent.getValue().toString());
399 }
400 }
401 else if (ent.getValue() instanceof List) {
402 if (ent.getValue() != null && !((List<?>)ent.getValue()).isEmpty()
403 && ((List<?>)ent.getValue()).get(0) != null &&
404 ((List<?>)ent.getValue()).get(0) instanceof TezWork.Dependency) {
405 if (out != null) {
406 boolean isFirst = true;
407 for (TezWork.Dependency dep: (List<TezWork.Dependency>)ent.getValue()) {
408 if (!isFirst) {
409 out.print(", ");
410 } else {
411 out.print("<- ");
412 isFirst = false;
413 }
414 out.print(dep.getName());
415 out.print(" (");
416 out.print(dep.getType());
417 out.print(")");
418 }
419 out.println();
420 }
421 if (jsonOutput) {
422 for (TezWork.Dependency dep: (List<TezWork.Dependency>)ent.getValue()) {
423 JSONObject jsonDep = new JSONObject(new LinkedHashMap<>());
424 jsonDep.put("parent", dep.getName());
425 jsonDep.put("type", dep.getType());
426 json.accumulate(ent.getKey().toString(), jsonDep);
427 }
428 }
429 } else if (ent.getValue() != null && !((List<?>) ent.getValue()).isEmpty()
430 && ((List<?>) ent.getValue()).get(0) != null &&
431 ((List<?>) ent.getValue()).get(0) instanceof SparkWork.Dependency) {
432 if (out != null) {
433 boolean isFirst = true;
434 for (SparkWork.Dependency dep: (List<SparkWork.Dependency>) ent.getValue()) {
435 if (!isFirst) {
436 out.print(", ");
437 } else {
438 out.print("<- ");
439 isFirst = false;
440 }
441 out.print(dep.getName());
442 out.print(" (");
443 out.print(dep.getShuffleType());
444 out.print(", ");
445 out.print(dep.getNumPartitions());
446 out.print(")");
447 }
448 out.println();
449 }
450 if (jsonOutput) {
451 for (SparkWork.Dependency dep: (List<SparkWork.Dependency>) ent.getValue()) {
452 JSONObject jsonDep = new JSONObject(new LinkedHashMap<>());
453 jsonDep.put("parent", dep.getName());
454 jsonDep.put("type", dep.getShuffleType());
455 jsonDep.put("partitions", dep.getNumPartitions());
456 json.accumulate(ent.getKey().toString(), jsonDep);
457 }
458 }
459 } else {
460 if (out != null) {
461 out.print(ent.getValue().toString());
462 out.println();
463 }
464 if (jsonOutput) {
465 json.put(ent.getKey().toString(), ent.getValue().toString());
466 }
467 }
468 }
469 else if (ent.getValue() instanceof Map) {
470 String stringValue = getBasictypeKeyedMap((Map)ent.getValue()).toString();
471 if (out != null) {
472 out.print(stringValue);
473 out.println();
474 }
475 if (jsonOutput) {
476 json.put(ent.getKey().toString(), stringValue);
477 }
478 }
479 else if (ent.getValue() != null) {
480 if (out != null) {
481 out.println();
482 }
483 JSONObject jsonOut = outputPlan(ent.getValue(), out,
484 extended, jsonOutput, jsonOutput ? 0 : indent + 2);
485 if (jsonOutput) {
486 json.put(ent.getKey().toString(), jsonOut);
487 }
488 }
489 else {
490 if (out != null) {
491 out.println();
492 }
493 }
494 }
495
496 return jsonOutput ? json : null;
497 }
498
499 /**
500 * Retruns a map which have either primitive or string keys.
501 *
502 * This is neccessary to discard object level comparators which may sort the objects based on some non-trivial logic.
503 *
504 * @param mp
505 * @return
506 */
507 private TreeMap<Object, Object> getBasictypeKeyedMap(Map<?, ?> mp) {
508 TreeMap<Object, Object> ret = new TreeMap<Object, Object>();
509 if (mp.size() > 0) {
510 Object firstKey = mp.keySet().iterator().next();
511 if (firstKey.getClass().isPrimitive() || firstKey instanceof String) {
512 // keep it as-is
513 ret.putAll(mp);
514 return ret;
515 } else {
516 for (Entry<?, ?> entry : mp.entrySet()) {
517 // discard possibly type related sorting order and replace with alphabetical
518 ret.put(entry.getKey().toString(), entry.getValue());
519 }
520 }
521 }
522 return ret;
523 }
524
525 private JSONArray outputList(List<?> l, PrintStream out, boolean hasHeader,
526 boolean extended, boolean jsonOutput, int indent) throws Exception {
527
528 boolean first_el = true;
529 boolean nl = false;
530 JSONArray outputArray = new JSONArray();
531
532 for (Object o : l) {
533 if (isPrintable(o)) {
534 String delim = first_el ? " " : ", ";
535 if (out != null) {
536 out.print(delim);
537 out.print(o);
538 }
539
540 if (jsonOutput) {
541 outputArray.put(o);
542 }
543 nl = true;
544 }
545 else {
546 if (first_el && (out != null) && hasHeader) {
547 out.println();
548 }
549 JSONObject jsonOut = outputPlan(o, out, extended,
550 jsonOutput, jsonOutput ? 0 : (hasHeader ? indent + 2 : indent));
551 if (jsonOutput) {
552 outputArray.put(jsonOut);
553 }
554 }
555
556 first_el = false;
557 }
558
559 if (nl && (out != null)) {
560 out.println();
561 }
562
563 return jsonOutput ? outputArray : null;
564 }
565
566 private boolean isPrintable(Object val) {
567 if (val instanceof Boolean || val instanceof String
568 || val instanceof Integer || val instanceof Long || val instanceof Byte
569 || val instanceof Float || val instanceof Double || val instanceof Path) {
570 return true;
571 }
572
573 if (val != null && val.getClass().isPrimitive()) {
574 return true;
575 }
576
577 return false;
578 }
579
580 private JSONObject outputPlan(Object work,
581 PrintStream out, boolean extended, boolean jsonOutput, int indent) throws Exception {
582 return outputPlan(work, out, extended, jsonOutput, indent, "");
583 }
584
585 private JSONObject outputPlan(Object work, PrintStream out,
586 boolean extended, boolean jsonOutput, int indent, String appendToHeader) throws Exception {
587 // Check if work has an explain annotation
588 Annotation note = AnnotationUtils.getAnnotation(work.getClass(), Explain.class);
589
590 String keyJSONObject = null;
591
592 if (note instanceof Explain) {
593 Explain xpl_note = (Explain) note;
594 boolean invokeFlag = false;
595 if (this.work != null && this.work.isUserLevelExplain()) {
596 invokeFlag = Level.USER.in(xpl_note.explainLevels());
597 } else {
598 if (extended) {
599 invokeFlag = Level.EXTENDED.in(xpl_note.explainLevels());
600 } else {
601 invokeFlag = Level.DEFAULT.in(xpl_note.explainLevels());
602 }
603 }
604 if (invokeFlag) {
605 keyJSONObject = xpl_note.displayName();
606 if (out != null) {
607 out.print(indentString(indent));
608 if (appendToHeader != null && !appendToHeader.isEmpty()) {
609 out.println(xpl_note.displayName() + appendToHeader);
610 } else {
611 out.println(xpl_note.displayName());
612 }
613 }
614 }
615 }
616
617 JSONObject json = jsonOutput ? new JSONObject(new LinkedHashMap<>()) : null;
618 // If this is an operator then we need to call the plan generation on the
619 // conf and then the children
620 if (work instanceof Operator) {
621 Operator<? extends OperatorDesc> operator =
622 (Operator<? extends OperatorDesc>) work;
623 if (operator.getConf() != null) {
624 String appender = isLogical ? " (" + operator.getOperatorId() + ")" : "";
625 JSONObject jsonOut = outputPlan(operator.getConf(), out, extended,
626 jsonOutput, jsonOutput ? 0 : indent, appender);
627 if (this.work != null && this.work.isUserLevelExplain()) {
628 if (jsonOut != null && jsonOut.length() > 0) {
629 ((JSONObject) jsonOut.get(JSONObject.getNames(jsonOut)[0])).put("OperatorId:",
630 operator.getOperatorId());
631 }
632 }
633 if (jsonOutput) {
634 json = jsonOut;
635 }
636 }
637
638 if (!visitedOps.contains(operator) || !isLogical) {
639 visitedOps.add(operator);
640 if (operator.getChildOperators() != null) {
641 int cindent = jsonOutput ? 0 : indent + 2;
642 for (Operator<? extends OperatorDesc> op : operator.getChildOperators()) {
643 JSONObject jsonOut = outputPlan(op, out, extended, jsonOutput, cindent);
644 if (jsonOutput) {
645 ((JSONObject)json.get(JSONObject.getNames(json)[0])).accumulate("children", jsonOut);
646 }
647 }
648 }
649 }
650
651 if (jsonOutput) {
652 return json;
653 }
654 return null;
655 }
656
657 // We look at all methods that generate values for explain
658 Method[] methods = work.getClass().getMethods();
659 Arrays.sort(methods, new MethodComparator());
660
661 for (Method m : methods) {
662 int prop_indents = jsonOutput ? 0 : indent + 2;
663 note = AnnotationUtils.getAnnotation(m, Explain.class);
664
665 if (note instanceof Explain) {
666 Explain xpl_note = (Explain) note;
667 boolean invokeFlag = false;
668 if (this.work != null && this.work.isUserLevelExplain()) {
669 invokeFlag = Level.USER.in(xpl_note.explainLevels());
670 } else {
671 if (extended) {
672 invokeFlag = Level.EXTENDED.in(xpl_note.explainLevels());
673 } else {
674 invokeFlag = Level.DEFAULT.in(xpl_note.explainLevels());
675 }
676 }
677 if (invokeFlag) {
678
679 Object val = null;
680 try {
681 val = m.invoke(work);
682 }
683 catch (InvocationTargetException ex) {
684 // Ignore the exception, this may be caused by external jars
685 val = null;
686 }
687
688 if (val == null) {
689 continue;
690 }
691
692 String header = null;
693 boolean skipHeader = xpl_note.skipHeader();
694 boolean emptyHeader = false;
695
696 if (!xpl_note.displayName().equals("")) {
697 header = indentString(prop_indents) + xpl_note.displayName() + ":";
698 }
699 else {
700 emptyHeader = true;
701 prop_indents = indent;
702 header = indentString(prop_indents);
703 }
704
705 // Try the output as a primitive object
706 if (isPrintable(val)) {
707 if (out != null && shouldPrint(xpl_note, val)) {
708 if (!skipHeader) {
709 out.print(header);
710 out.print(" ");
711 }
712 out.println(val);
713 }
714 if (jsonOutput && shouldPrint(xpl_note, val)) {
715 json.put(header, val.toString());
716 }
717 continue;
718 }
719
720 int ind = 0;
721 if (!jsonOutput) {
722 if (!skipHeader) {
723 ind = prop_indents + 2;
724 } else {
725 ind = indent;
726 }
727 }
728
729 // Try this as a map
730 if (val instanceof Map) {
731 // Go through the map and print out the stuff
732 Map<?, ?> mp = (Map<?, ?>) val;
733
734 if (out != null && !skipHeader && mp != null && !mp.isEmpty()) {
735 out.print(header);
736 }
737
738 JSONObject jsonOut = outputMap(mp, !skipHeader && !emptyHeader, out, extended, jsonOutput, ind);
739 if (jsonOutput && !mp.isEmpty()) {
740 json.put(header, jsonOut);
741 }
742 continue;
743 }
744
745 // Try this as a list
746 if (val instanceof List || val instanceof Set) {
747 List l = val instanceof List ? (List)val : new ArrayList((Set)val);
748 if (out != null && !skipHeader && l != null && !l.isEmpty()) {
749 out.print(header);
750 }
751
752 JSONArray jsonOut = outputList(l, out, !skipHeader && !emptyHeader, extended, jsonOutput, ind);
753
754 if (jsonOutput && !l.isEmpty()) {
755 json.put(header, jsonOut);
756 }
757
758 continue;
759 }
760
761 // Finally check if it is serializable
762 try {
763 if (!skipHeader && out != null) {
764 out.println(header);
765 }
766 JSONObject jsonOut = outputPlan(val, out, extended, jsonOutput, ind);
767 if (jsonOutput && jsonOut != null && jsonOut.length() != 0) {
768 if (!skipHeader) {
769 json.put(header, jsonOut);
770 } else {
771 for(String k: JSONObject.getNames(jsonOut)) {
772 json.put(k, jsonOut.get(k));
773 }
774 }
775 }
776 continue;
777 }
778 catch (ClassCastException ce) {
779 // Ignore
780 }
781 }
782 }
783 }
784
785 if (jsonOutput) {
786 if (keyJSONObject != null) {
787 JSONObject ret = new JSONObject(new LinkedHashMap<>());
788 ret.put(keyJSONObject, json);
789 return ret;
790 }
791
792 return json;
793 }
794 return null;
795 }
796
797 /**
798 * use case: we want to print the object in explain only if it is true
799 * how to do : print it unless the following 3 are all true:
800 * 1. displayOnlyOnTrue tag is on
801 * 2. object is boolean
802 * 3. object is false
803 * @param exp
804 * @param val
805 * @return
806 */
807 private boolean shouldPrint(Explain exp, Object val) {
808 if (exp.displayOnlyOnTrue() && (val instanceof Boolean) & !((Boolean)val)) {
809 return false;
810 }
811 return true;
812 }
813
814 private JSONObject outputPlan(Task<?> task,
815 PrintStream out, JSONObject parentJSON, boolean extended,
816 boolean jsonOutput, int indent) throws Exception {
817
818 if (out != null) {
819 out.print(indentString(indent));
820 out.print("Stage: ");
821 out.print(task.getId());
822 out.print("\n");
823 }
824
825 // Start by getting the work part of the task and call the output plan for
826 // the work
827 JSONObject jsonOutputPlan = outputPlan(task.getWork(), out, extended,
828 jsonOutput, jsonOutput ? 0 : indent + 2);
829
830 if (out != null) {
831 out.println();
832 }
833
834 if (jsonOutput) {
835 parentJSON.put(task.getId(), jsonOutputPlan);
836 }
837 return null;
838 }
839
840 private JSONObject outputDependencies(Task<?> task,
841 PrintStream out, JSONObject parentJson, boolean jsonOutput, boolean taskType, int indent)
842 throws Exception {
843
844 boolean first = true;
845 JSONObject json = jsonOutput ? new JSONObject(new LinkedHashMap<>()) : null;
846 if (out != null) {
847 out.print(indentString(indent));
848 out.print(task.getId());
849 }
850
851 if ((task.getParentTasks() == null || task.getParentTasks().isEmpty())) {
852 if (task.isRootTask()) {
853 if (out != null) {
854 out.print(" is a root stage");
855 }
856
857 if (jsonOutput) {
858 json.put("ROOT STAGE", "TRUE");
859 }
860 }
861 }
862 else {
863 StringBuilder s = new StringBuilder();
864 first = true;
865 for (Task<?> parent : task.getParentTasks()) {
866 if (!first) {
867 s.append(", ");
868 }
869 first = false;
870 s.append(parent.getId());
871 }
872
873 if (out != null) {
874 out.print(" depends on stages: ");
875 out.print(s.toString());
876 }
877 if (jsonOutput) {
878 json.put("DEPENDENT STAGES", s.toString());
879 }
880 }
881
882 Task<?> currBackupTask = task.getBackupTask();
883 if (currBackupTask != null) {
884 if (out != null) {
885 out.print(" has a backup stage: ");
886 out.print(currBackupTask.getId());
887 }
888 if (jsonOutput) {
889 json.put("BACKUP STAGE", currBackupTask.getId());
890 }
891 }
892
893 if (task instanceof ConditionalTask
894 && ((ConditionalTask) task).getListTasks() != null) {
895 StringBuilder s = new StringBuilder();
896 first = true;
897 for (Task<?> con : ((ConditionalTask) task).getListTasks()) {
898 if (!first) {
899 s.append(", ");
900 }
901 first = false;
902 s.append(con.getId());
903 }
904
905 if (out != null) {
906 out.print(" , consists of ");
907 out.print(s.toString());
908 }
909 if (jsonOutput) {
910 json.put("CONDITIONAL CHILD TASKS", s.toString());
911 }
912 }
913 if (taskType) {
914 if (out != null) {
915 out.print(" [");
916 out.print(task.getType());
917 out.print("]");
918 }
919 if (jsonOutput) {
920 json.put("TASK TYPE", task.getType().name());
921 }
922 }
923
924 if (out != null) {
925 out.println();
926 }
927 return jsonOutput ? json : null;
928 }
929
930 public String outputAST(String treeString, PrintStream out,
931 boolean jsonOutput, int indent) throws JSONException {
932 if (out != null) {
933 out.print(indentString(indent));
934 out.println("ABSTRACT SYNTAX TREE:");
935 out.print(indentString(indent + 2));
936 out.println(treeString);
937 }
938
939 return jsonOutput ? treeString : null;
940 }
941
942 public JSONObject outputDependencies(PrintStream out, boolean jsonOutput,
943 boolean appendTaskType, List<Task> tasks)
944 throws Exception {
945
946 if (out != null) {
947 out.println("STAGE DEPENDENCIES:");
948 }
949
950 JSONObject json = jsonOutput ? new JSONObject(new LinkedHashMap<>()) : null;
951 for (Task task : tasks) {
952 JSONObject jsonOut = outputDependencies(task, out, json, jsonOutput, appendTaskType, 2);
953 if (jsonOutput && jsonOut != null) {
954 json.put(task.getId(), jsonOut);
955 }
956 }
957
958 return jsonOutput ? json : null;
959 }
960
961 public JSONObject outputStagePlans(PrintStream out, List<Task> tasks,
962 boolean jsonOutput, boolean isExtended)
963 throws Exception {
964
965 if (out != null) {
966 out.println("STAGE PLANS:");
967 }
968
969 JSONObject json = jsonOutput ? new JSONObject(new LinkedHashMap<>()) : null;
970 for (Task task : tasks) {
971 outputPlan(task, out, json, isExtended, jsonOutput, 2);
972 }
973 return jsonOutput ? json : null;
974 }
975
976 /**
977 * MethodComparator.
978 *
979 */
980 public class MethodComparator implements Comparator<Method> {
981 @Override
982 public int compare(Method m1, Method m2) {
983 return m1.getName().compareTo(m2.getName());
984 }
985 }
986
987 @Override
988 public StageType getType() {
989 return StageType.EXPLAIN;
990 }
991
992 @Override
993 public String getName() {
994 return "EXPLAIN";
995 }
996
997 public List<FieldSchema> getResultSchema() {
998 FieldSchema tmpFieldSchema = new FieldSchema();
999 List<FieldSchema> colList = new ArrayList<FieldSchema>();
1000
1001 tmpFieldSchema.setName(EXPL_COLUMN_NAME);
1002 tmpFieldSchema.setType(STRING_TYPE_NAME);
1003
1004 colList.add(tmpFieldSchema);
1005 return colList;
1006 }
1007 }