YARN-8048. Support auto-spawning of admin configured services during bootstrap of...
[hadoop.git] / hadoop-yarn-project / hadoop-yarn / hadoop-yarn-api / src / main / java / org / apache / hadoop / yarn / conf / YarnConfiguration.java
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 package org.apache.hadoop.yarn.conf;
20
21 import java.net.InetSocketAddress;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.Collection;
25 import java.util.Collections;
26 import java.util.List;
27
28 import org.apache.hadoop.HadoopIllegalArgumentException;
29 import org.apache.hadoop.classification.InterfaceAudience.Private;
30 import org.apache.hadoop.classification.InterfaceAudience.Public;
31 import org.apache.hadoop.classification.InterfaceStability.Evolving;
32 import org.apache.hadoop.classification.InterfaceStability.Unstable;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.CommonConfigurationKeys;
35 import org.apache.hadoop.ha.ActiveStandbyElector;
36 import org.apache.hadoop.http.HttpConfig;
37 import org.apache.hadoop.net.NetUtils;
38 import org.apache.hadoop.util.BasicDiskValidator;
39 import org.apache.hadoop.util.StringUtils;
40 import org.apache.hadoop.yarn.api.ApplicationConstants;
41
42 @Public
43 @Evolving
44 public class YarnConfiguration extends Configuration {
45
46 @Private
47 public static final String DR_CONFIGURATION_FILE= "dynamic-resources.xml";
48
49 @Private
50 public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml";
51
52 @Private
53 public static final String HADOOP_POLICY_CONFIGURATION_FILE =
54 "hadoop-policy.xml";
55
56 @Private
57 public static final String YARN_SITE_CONFIGURATION_FILE = "yarn-site.xml";
58
59 private static final String YARN_DEFAULT_CONFIGURATION_FILE =
60 "yarn-default.xml";
61
62 @Private
63 public static final String CORE_SITE_CONFIGURATION_FILE = "core-site.xml";
64
65 @Private
66 public static final String RESOURCE_TYPES_CONFIGURATION_FILE =
67 "resource-types.xml";
68
69 @Private
70 public static final String NODE_RESOURCES_CONFIGURATION_FILE =
71 "node-resources.xml";
72
73 @Private
74 public static final List<String> RM_CONFIGURATION_FILES =
75 Collections.unmodifiableList(Arrays.asList(
76 RESOURCE_TYPES_CONFIGURATION_FILE,
77 DR_CONFIGURATION_FILE,
78 CS_CONFIGURATION_FILE,
79 HADOOP_POLICY_CONFIGURATION_FILE,
80 YARN_SITE_CONFIGURATION_FILE,
81 CORE_SITE_CONFIGURATION_FILE));
82
83 @Private
84 public static final List<String> NM_CONFIGURATION_FILES =
85 Collections.unmodifiableList(Arrays.asList(
86 NODE_RESOURCES_CONFIGURATION_FILE,
87 DR_CONFIGURATION_FILE,
88 CS_CONFIGURATION_FILE,
89 HADOOP_POLICY_CONFIGURATION_FILE,
90 YARN_SITE_CONFIGURATION_FILE,
91 CORE_SITE_CONFIGURATION_FILE));
92
93 @Evolving
94 public static final int APPLICATION_MAX_TAGS = 10;
95
96 @Evolving
97 public static final int APPLICATION_MAX_TAG_LENGTH = 100;
98
99 static {
100 addDeprecatedKeys();
101 Configuration.addDefaultResource(YARN_DEFAULT_CONFIGURATION_FILE);
102 Configuration.addDefaultResource(YARN_SITE_CONFIGURATION_FILE);
103 }
104
105 private static void addDeprecatedKeys() {
106 Configuration.addDeprecations(new DeprecationDelta[] {
107 new DeprecationDelta("yarn.client.max-nodemanagers-proxies",
108 NM_CLIENT_MAX_NM_PROXIES)
109 });
110 Configuration.addDeprecations(new DeprecationDelta[] {
111 new DeprecationDelta(RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
112 SYSTEM_METRICS_PUBLISHER_ENABLED),
113 new DeprecationDelta(RM_ZK_ACL, CommonConfigurationKeys.ZK_ACL),
114 new DeprecationDelta(RM_ZK_AUTH, CommonConfigurationKeys.ZK_AUTH),
115 new DeprecationDelta(RM_ZK_ADDRESS,
116 CommonConfigurationKeys.ZK_ADDRESS),
117 new DeprecationDelta(RM_ZK_NUM_RETRIES,
118 CommonConfigurationKeys.ZK_NUM_RETRIES),
119 new DeprecationDelta(RM_ZK_TIMEOUT_MS,
120 CommonConfigurationKeys.ZK_TIMEOUT_MS),
121 new DeprecationDelta(RM_ZK_RETRY_INTERVAL_MS,
122 CommonConfigurationKeys.ZK_RETRY_INTERVAL_MS),
123 });
124 }
125
126 //Configurations
127
128 public static final String YARN_PREFIX = "yarn.";
129
130 /////////////////////////////
131 // Resource types configs
132 ////////////////////////////
133
134 public static final String RESOURCE_TYPES =
135 YarnConfiguration.YARN_PREFIX + "resource-types";
136
137 public static final String NM_RESOURCES_PREFIX =
138 YarnConfiguration.NM_PREFIX + "resource-type.";
139
140 /** Delay before deleting resource to ease debugging of NM issues */
141 public static final String DEBUG_NM_DELETE_DELAY_SEC =
142 YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec";
143
144 public static final String NM_LOG_CONTAINER_DEBUG_INFO =
145 YarnConfiguration.NM_PREFIX + "log-container-debug-info.enabled";
146
147 public static final boolean DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO = true;
148
149 ////////////////////////////////
150 // IPC Configs
151 ////////////////////////////////
152 public static final String IPC_PREFIX = YARN_PREFIX + "ipc.";
153
154 /** Factory to create client IPC classes.*/
155 public static final String IPC_CLIENT_FACTORY_CLASS =
156 IPC_PREFIX + "client.factory.class";
157 public static final String DEFAULT_IPC_CLIENT_FACTORY_CLASS =
158 "org.apache.hadoop.yarn.factories.impl.pb.RpcClientFactoryPBImpl";
159
160 /** Factory to create server IPC classes.*/
161 public static final String IPC_SERVER_FACTORY_CLASS =
162 IPC_PREFIX + "server.factory.class";
163 public static final String DEFAULT_IPC_SERVER_FACTORY_CLASS =
164 "org.apache.hadoop.yarn.factories.impl.pb.RpcServerFactoryPBImpl";
165
166 /** Factory to create serializeable records.*/
167 public static final String IPC_RECORD_FACTORY_CLASS =
168 IPC_PREFIX + "record.factory.class";
169 public static final String DEFAULT_IPC_RECORD_FACTORY_CLASS =
170 "org.apache.hadoop.yarn.factories.impl.pb.RecordFactoryPBImpl";
171
172 /** RPC class implementation*/
173 public static final String IPC_RPC_IMPL =
174 IPC_PREFIX + "rpc.class";
175 public static final String DEFAULT_IPC_RPC_IMPL =
176 "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC";
177
178 ////////////////////////////////
179 // Resource Manager Configs
180 ////////////////////////////////
181 public static final String RM_PREFIX = "yarn.resourcemanager.";
182
183 public static final String RM_CLUSTER_ID = RM_PREFIX + "cluster-id";
184 public static final String DEFAULT_RM_CLUSTER_ID = "yarn_cluster";
185
186 public static final String RM_HOSTNAME = RM_PREFIX + "hostname";
187
188 public static final String RM_EPOCH = RM_PREFIX + "epoch";
189 public static final long DEFAULT_RM_EPOCH = 0L;
190
191 /** The address of the applications manager interface in the RM.*/
192 public static final String RM_ADDRESS =
193 RM_PREFIX + "address";
194 public static final int DEFAULT_RM_PORT = 8032;
195 public static final String DEFAULT_RM_ADDRESS =
196 "0.0.0.0:" + DEFAULT_RM_PORT;
197
198 public static final String RM_APPLICATION_MASTER_SERVICE_PROCESSORS =
199 RM_PREFIX + "application-master-service.processors";
200
201 public static final String RM_AUTO_UPDATE_CONTAINERS =
202 RM_PREFIX + "auto-update.containers";
203 public static final boolean DEFAULT_RM_AUTO_UPDATE_CONTAINERS = false;
204
205 /** The actual bind address for the RM.*/
206 public static final String RM_BIND_HOST =
207 RM_PREFIX + "bind-host";
208
209 /** The number of threads used to handle applications manager requests.*/
210 public static final String RM_CLIENT_THREAD_COUNT =
211 RM_PREFIX + "client.thread-count";
212 public static final int DEFAULT_RM_CLIENT_THREAD_COUNT = 50;
213
214 /** Number of threads used to launch/cleanup AM.*/
215 public static final String RM_AMLAUNCHER_THREAD_COUNT =
216 RM_PREFIX + "amlauncher.thread-count";
217 public static final int DEFAULT_RM_AMLAUNCHER_THREAD_COUNT = 50;
218
219 /** Retry times to connect with NM.*/
220 public static final String RM_NODEMANAGER_CONNECT_RETRIES =
221 RM_PREFIX + "nodemanager-connect-retries";
222 public static final int DEFAULT_RM_NODEMANAGER_CONNECT_RETRIES = 10;
223
224 /** The Kerberos principal for the resource manager.*/
225 public static final String RM_PRINCIPAL =
226 RM_PREFIX + "principal";
227
228 /** The address of the scheduler interface.*/
229 public static final String RM_SCHEDULER_ADDRESS =
230 RM_PREFIX + "scheduler.address";
231 public static final int DEFAULT_RM_SCHEDULER_PORT = 8030;
232 public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" +
233 DEFAULT_RM_SCHEDULER_PORT;
234
235 /** Miniumum request grant-able by the RM scheduler. */
236 public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_MB =
237 YARN_PREFIX + "scheduler.minimum-allocation-mb";
238 public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 1024;
239 public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES =
240 YARN_PREFIX + "scheduler.minimum-allocation-vcores";
241 public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 1;
242
243 /** Maximum request grant-able by the RM scheduler. */
244 public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_MB =
245 YARN_PREFIX + "scheduler.maximum-allocation-mb";
246 public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 8192;
247 public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES =
248 YARN_PREFIX + "scheduler.maximum-allocation-vcores";
249 public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 4;
250
251 /** Number of threads to handle scheduler interface.*/
252 public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT =
253 RM_PREFIX + "scheduler.client.thread-count";
254 public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50;
255
256 /** If the port should be included or not in the node name. The node name
257 * is used by the scheduler for resource requests allocation location
258 * matching. Typically this is just the hostname, using the port is needed
259 * when using minicluster and specific NM are required.*/
260 public static final String RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME =
261 YARN_PREFIX + "scheduler.include-port-in-node-name";
262 public static final boolean DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME =
263 false;
264
265 /** Configured scheduler queue placement rules. */
266 public static final String QUEUE_PLACEMENT_RULES = YARN_PREFIX
267 + "scheduler.queue-placement-rules";
268 /** UserGroupMappingPlacementRule configuration string. */
269 public static final String USER_GROUP_PLACEMENT_RULE = "user-group";
270
271 public static final String APP_NAME_PLACEMENT_RULE = "app-name";
272
273 /** Enable Resource Manager webapp ui actions */
274 public static final String RM_WEBAPP_UI_ACTIONS_ENABLED =
275 RM_PREFIX + "webapp.ui-actions.enabled";
276 public static final boolean DEFAULT_RM_WEBAPP_UI_ACTIONS_ENABLED =
277 true;
278
279 /** Whether the RM should enable Reservation System */
280 public static final String RM_RESERVATION_SYSTEM_ENABLE = RM_PREFIX
281 + "reservation-system.enable";
282 public static final boolean DEFAULT_RM_RESERVATION_SYSTEM_ENABLE = false;
283
284 /** The class to use as the Reservation System. */
285 public static final String RM_RESERVATION_SYSTEM_CLASS = RM_PREFIX
286 + "reservation-system.class";
287
288 /** The PlanFollower for the Reservation System. */
289 public static final String RM_RESERVATION_SYSTEM_PLAN_FOLLOWER = RM_PREFIX
290 + "reservation-system.plan.follower";
291
292 /** The step size of the Reservation System. */
293 public static final String RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP =
294 RM_PREFIX + "reservation-system.planfollower.time-step";
295 public static final long DEFAULT_RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP =
296 1000L;
297
298 /** The maximum periodicity for the Reservation System. */
299 public static final String RM_RESERVATION_SYSTEM_MAX_PERIODICITY =
300 RM_PREFIX + "reservation-system.max-periodicity";
301 public static final long DEFAULT_RM_RESERVATION_SYSTEM_MAX_PERIODICITY =
302 86400000L;
303
304 /**
305 * Enable periodic monitor threads.
306 * @see #RM_SCHEDULER_MONITOR_POLICIES
307 */
308 public static final String RM_SCHEDULER_ENABLE_MONITORS =
309 RM_PREFIX + "scheduler.monitor.enable";
310 public static final boolean DEFAULT_RM_SCHEDULER_ENABLE_MONITORS = false;
311
312 /** List of SchedulingEditPolicy classes affecting the scheduler. */
313 public static final String RM_SCHEDULER_MONITOR_POLICIES =
314 RM_PREFIX + "scheduler.monitor.policies";
315
316 /** The address of the RM web application.*/
317 public static final String RM_WEBAPP_ADDRESS =
318 RM_PREFIX + "webapp.address";
319
320 public static final int DEFAULT_RM_WEBAPP_PORT = 8088;
321 public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:" +
322 DEFAULT_RM_WEBAPP_PORT;
323
324 /** The https address of the RM web application.*/
325 public static final String RM_WEBAPP_HTTPS_ADDRESS =
326 RM_PREFIX + "webapp.https.address";
327 public static final boolean YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT = false;
328 public static final String YARN_SSL_SERVER_RESOURCE_DEFAULT = "ssl-server.xml";
329
330 public static final int DEFAULT_RM_WEBAPP_HTTPS_PORT = 8090;
331 public static final String DEFAULT_RM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:"
332 + DEFAULT_RM_WEBAPP_HTTPS_PORT;
333
334 /**
335 * Enable YARN WebApp V2.
336 */
337 public static final String YARN_WEBAPP_UI2_ENABLE = "yarn."
338 + "webapp.ui2.enable";
339 public static final boolean DEFAULT_YARN_WEBAPP_UI2_ENABLE = false;
340
341 public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn."
342 + "webapp.ui2.war-file-path";
343 public static final String YARN_API_SERVICES_ENABLE = "yarn."
344 + "webapp.api-service.enable";
345
346 @Private
347 public static final String DEFAULT_YARN_API_SYSTEM_SERVICES_CLASS =
348 "org.apache.hadoop.yarn.service.client.SystemServiceManagerImpl";
349
350 public static final String RM_RESOURCE_TRACKER_ADDRESS =
351 RM_PREFIX + "resource-tracker.address";
352 public static final int DEFAULT_RM_RESOURCE_TRACKER_PORT = 8031;
353 public static final String DEFAULT_RM_RESOURCE_TRACKER_ADDRESS =
354 "0.0.0.0:" + DEFAULT_RM_RESOURCE_TRACKER_PORT;
355
356 /** The expiry interval for application master reporting.*/
357 public static final String RM_AM_EXPIRY_INTERVAL_MS =
358 YARN_PREFIX + "am.liveness-monitor.expiry-interval-ms";
359 public static final int DEFAULT_RM_AM_EXPIRY_INTERVAL_MS = 600000;
360
361 /** How long to wait until a node manager is considered dead.*/
362 public static final String RM_NM_EXPIRY_INTERVAL_MS =
363 YARN_PREFIX + "nm.liveness-monitor.expiry-interval-ms";
364 public static final int DEFAULT_RM_NM_EXPIRY_INTERVAL_MS = 600000;
365
366 /** Are acls enabled.*/
367 public static final String YARN_ACL_ENABLE =
368 YARN_PREFIX + "acl.enable";
369 public static final boolean DEFAULT_YARN_ACL_ENABLE = false;
370
371 /** Are reservation acls enabled.*/
372 public static final String YARN_RESERVATION_ACL_ENABLE =
373 YARN_PREFIX + "acl.reservation-enable";
374 public static final boolean DEFAULT_YARN_RESERVATION_ACL_ENABLE = false;
375
376 public static boolean isAclEnabled(Configuration conf) {
377 return conf.getBoolean(YARN_ACL_ENABLE, DEFAULT_YARN_ACL_ENABLE);
378 }
379
380 /** ACL of who can be admin of YARN cluster.*/
381 public static final String YARN_ADMIN_ACL =
382 YARN_PREFIX + "admin.acl";
383 public static final String DEFAULT_YARN_ADMIN_ACL = "*";
384
385 /** ACL used in case none is found. Allows nothing. */
386 public static final String DEFAULT_YARN_APP_ACL = " ";
387
388 /** Setting that controls whether opportunistic container allocation
389 * is enabled or not. */
390 @Unstable
391 public static final String OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED =
392 RM_PREFIX + "opportunistic-container-allocation.enabled";
393 public static final boolean
394 DEFAULT_OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED = false;
395
396 /** Number of nodes to be used by the Opportunistic Container allocator for
397 * dispatching containers during container allocation. */
398 @Unstable
399 public static final String OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED =
400 RM_PREFIX + "opportunistic-container-allocation.nodes-used";
401 public static final int DEFAULT_OPP_CONTAINER_ALLOCATION_NODES_NUMBER_USED =
402 10;
403
404 /** Frequency for computing least loaded NMs. */
405 @Unstable
406 public static final String NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS =
407 RM_PREFIX + "nm-container-queuing.sorting-nodes-interval-ms";
408 public static final long
409 DEFAULT_NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS = 1000;
410
411 /** Comparator for determining node load for scheduling of opportunistic
412 * containers. */
413 @Unstable
414 public static final String NM_CONTAINER_QUEUING_LOAD_COMPARATOR =
415 RM_PREFIX + "nm-container-queuing.load-comparator";
416 public static final String DEFAULT_NM_CONTAINER_QUEUING_LOAD_COMPARATOR =
417 "QUEUE_LENGTH";
418
419 /** Value of standard deviation used for calculation of queue limit
420 * thresholds. */
421 @Unstable
422 public static final String NM_CONTAINER_QUEUING_LIMIT_STDEV =
423 RM_PREFIX + "nm-container-queuing.queue-limit-stdev";
424 public static final float DEFAULT_NM_CONTAINER_QUEUING_LIMIT_STDEV =
425 1.0f;
426
427 /** Min length of container queue at NodeManager. This is a cluster-wide
428 * configuration that acts as the lower-bound of optimal queue length
429 * calculated by the NodeQueueLoadMonitor */
430 @Unstable
431 public static final String NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH =
432 RM_PREFIX + "nm-container-queuing.min-queue-length";
433 public static final int DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_LENGTH = 5;
434
435 /** Max length of container queue at NodeManager. This is a cluster-wide
436 * configuration that acts as the upper-bound of optimal queue length
437 * calculated by the NodeQueueLoadMonitor */
438 @Unstable
439 public static final String NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH =
440 RM_PREFIX + "nm-container-queuing.max-queue-length";
441 public static final int DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_LENGTH = 15;
442
443 /** Min queue wait time for a container at a NodeManager. */
444 @Unstable
445 public static final String NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS =
446 RM_PREFIX + "nm-container-queuing.min-queue-wait-time-ms";
447 public static final int DEFAULT_NM_CONTAINER_QUEUING_MIN_QUEUE_WAIT_TIME_MS =
448 10;
449
450 /** Max queue wait time for a container queue at a NodeManager. */
451 @Unstable
452 public static final String NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS =
453 RM_PREFIX + "nm-container-queuing.max-queue-wait-time-ms";
454 public static final int DEFAULT_NM_CONTAINER_QUEUING_MAX_QUEUE_WAIT_TIME_MS =
455 100;
456
457 /**
458 * Enable/disable intermediate-data encryption at YARN level. For now, this
459 * only is used by the FileSystemRMStateStore to setup right file-system
460 * security attributes.
461 */
462 @Private
463 public static final String YARN_INTERMEDIATE_DATA_ENCRYPTION = YARN_PREFIX
464 + "intermediate-data-encryption.enable";
465
466 @Private
467 public static final boolean DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION = false;
468
469 /** The address of the RM admin interface.*/
470 public static final String RM_ADMIN_ADDRESS =
471 RM_PREFIX + "admin.address";
472 public static final int DEFAULT_RM_ADMIN_PORT = 8033;
473 public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:" +
474 DEFAULT_RM_ADMIN_PORT;
475
476 /**Number of threads used to handle RM admin interface.*/
477 public static final String RM_ADMIN_CLIENT_THREAD_COUNT =
478 RM_PREFIX + "admin.client.thread-count";
479 public static final int DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT = 1;
480
481 /**
482 * The maximum number of application attempts.
483 * It's a global setting for all application masters.
484 */
485 public static final String RM_AM_MAX_ATTEMPTS =
486 RM_PREFIX + "am.max-attempts";
487 public static final int DEFAULT_RM_AM_MAX_ATTEMPTS = 2;
488
489 /** The keytab for the resource manager.*/
490 public static final String RM_KEYTAB =
491 RM_PREFIX + "keytab";
492
493 /**The kerberos principal to be used for spnego filter for RM.*/
494 public static final String RM_WEBAPP_SPNEGO_USER_NAME_KEY =
495 RM_PREFIX + "webapp.spnego-principal";
496
497 /**The kerberos keytab to be used for spnego filter for RM.*/
498 public static final String RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
499 RM_PREFIX + "webapp.spnego-keytab-file";
500
501 /**
502 * Flag to enable override of the default kerberos authentication filter with
503 * the RM authentication filter to allow authentication using delegation
504 * tokens(fallback to kerberos if the tokens are missing). Only applicable
505 * when the http authentication type is kerberos.
506 */
507 public static final String RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER = RM_PREFIX
508 + "webapp.delegation-token-auth-filter.enabled";
509 public static final boolean DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER =
510 true;
511
512 /** Enable cross origin (CORS) support. **/
513 public static final String RM_WEBAPP_ENABLE_CORS_FILTER =
514 RM_PREFIX + "webapp.cross-origin.enabled";
515 public static final boolean DEFAULT_RM_WEBAPP_ENABLE_CORS_FILTER = false;
516
517 /** How long to wait until a container is considered dead.*/
518 public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS =
519 RM_PREFIX + "rm.container-allocation.expiry-interval-ms";
520 public static final int DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 600000;
521
522 /** Path to file with nodes to include.*/
523 public static final String RM_NODES_INCLUDE_FILE_PATH =
524 RM_PREFIX + "nodes.include-path";
525 public static final String DEFAULT_RM_NODES_INCLUDE_FILE_PATH = "";
526
527 /** Path to file with nodes to exclude.*/
528 public static final String RM_NODES_EXCLUDE_FILE_PATH =
529 RM_PREFIX + "nodes.exclude-path";
530 public static final String DEFAULT_RM_NODES_EXCLUDE_FILE_PATH = "";
531
532 /** Number of threads to handle resource tracker calls.*/
533 public static final String RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT =
534 RM_PREFIX + "resource-tracker.client.thread-count";
535 public static final int DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 50;
536
537 /** The class to use as the resource scheduler.*/
538 public static final String RM_SCHEDULER =
539 RM_PREFIX + "scheduler.class";
540
541 /**
542 * Specify which handler will be used to process PlacementConstraints.
543 * For details on PlacementConstraints, please refer to
544 * {@link org.apache.hadoop.yarn.api.resource.PlacementConstraint}
545 */
546 @Private
547 public static final String RM_PLACEMENT_CONSTRAINTS_HANDLER =
548 RM_PREFIX + "placement-constraints.handler";
549
550 /**
551 * This handler rejects all allocate calls made by an application, if they
552 * contain a {@link org.apache.hadoop.yarn.api.records.SchedulingRequest}.
553 */
554 @Private
555 public static final String DISABLED_RM_PLACEMENT_CONSTRAINTS_HANDLER =
556 "disabled";
557
558 /**
559 * Using this handler, the placement of containers with constraints is
560 * determined as a pre-processing step before the capacity or the fair
561 * scheduler is called. Once the placement is decided, the capacity/fair
562 * scheduler is invoked to perform the actual allocation. The advantage of
563 * this approach is that it supports all constraint types (affinity,
564 * anti-affinity, cardinality). Moreover, it considers multiple containers at
565 * a time, which allows to satisfy more constraints than a container-at-a-time
566 * approach can achieve. As it sits outside the main scheduler, it can be used
567 * by both the capacity and fair schedulers. Note that at the moment it does
568 * not account for task priorities within an application, given that such
569 * priorities might be conflicting with the placement constraints.
570 */
571 @Private
572 public static final String PROCESSOR_RM_PLACEMENT_CONSTRAINTS_HANDLER =
573 "placement-processor";
574
575 /**
576 * Using this handler, containers with constraints will be placed by the main
577 * scheduler. If the configured RM scheduler
578 * <pre>yarn.resourcemanager.scheduler.class</pre>
579 * cannot handle placement constraints, the corresponding SchedulingRequests
580 * will be rejected. As of now, only the capacity scheduler supports
581 * SchedulingRequests. In particular, it currently supports anti-affinity
582 * constraints (no affinity or cardinality) and places one container at a
583 * time. The advantage of this handler compared to the placement-processor is
584 * that it follows the same ordering rules for queues (sorted by utilization,
585 * priority) and apps (sorted by FIFO/fairness/priority) as the ones followed
586 * by the main scheduler.
587 */
588 @Private
589 public static final String
590 SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER =
591 "scheduler";
592
593 /** Placement Algorithm. */
594 public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_CLASS =
595 RM_PREFIX + "placement-constraints.algorithm.class";
596
597 /** Used for BasicPlacementAlgorithm - default SERIAL. **/
598 public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_ITERATOR =
599 RM_PREFIX + "placement-constraints.algorithm.iterator";
600
601 public static final String RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS =
602 RM_PREFIX + "placement-constraints.retry-attempts";
603
604 public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS = 3;
605
606 public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE =
607 RM_PREFIX + "placement-constraints.algorithm.pool-size";
608
609 public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE =
610 1;
611
612 public static final String RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE =
613 RM_PREFIX + "placement-constraints.scheduler.pool-size";
614
615 public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE =
616 1;
617
618 public static final String DEFAULT_RM_SCHEDULER =
619 "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler";
620
621 /** RM set next Heartbeat interval for NM */
622 public static final String RM_NM_HEARTBEAT_INTERVAL_MS =
623 RM_PREFIX + "nodemanagers.heartbeat-interval-ms";
624 public static final long DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS = 1000;
625
626 /** Number of worker threads that write the history data. */
627 public static final String RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE =
628 RM_PREFIX + "history-writer.multi-threaded-dispatcher.pool-size";
629 public static final int DEFAULT_RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE =
630 10;
631
632 /**
633 * The setting that controls whether yarn system metrics is published on the
634 * timeline server or not by RM. This configuration setting is for ATS V1.
635 * This is now deprecated in favor of SYSTEM_METRICS_PUBLISHER_ENABLED.
636 */
637 public static final String RM_SYSTEM_METRICS_PUBLISHER_ENABLED = RM_PREFIX
638 + "system-metrics-publisher.enabled";
639 public static final boolean DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED =
640 false;
641
642 /**
643 * The setting that controls whether yarn system metrics is published on the
644 * timeline server or not by RM and NM. This configuration setting is for
645 * ATS v2.
646 */
647 public static final String SYSTEM_METRICS_PUBLISHER_ENABLED = YARN_PREFIX
648 + "system-metrics-publisher.enabled";
649 public static final boolean DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED = false;
650
651 /**
652 * The setting that controls whether yarn container events are published to
653 * the timeline service or not by RM. This configuration setting is for ATS
654 * V2
655 */
656 public static final String RM_PUBLISH_CONTAINER_EVENTS_ENABLED = YARN_PREFIX
657 + "rm.system-metrics-publisher.emit-container-events";
658 public static final boolean DEFAULT_RM_PUBLISH_CONTAINER_EVENTS_ENABLED =
659 false;
660
661 public static final String RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =
662 RM_PREFIX + "system-metrics-publisher.dispatcher.pool-size";
663 public static final int
664 DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 10;
665
666 //RM delegation token related keys
667 public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY =
668 RM_PREFIX + "delegation.key.update-interval";
669 public static final long RM_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT =
670 24*60*60*1000; // 1 day
671 public static final String RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY =
672 RM_PREFIX + "delegation.token.renew-interval";
673 public static final long RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT =
674 24*60*60*1000; // 1 day
675 public static final String RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY =
676 RM_PREFIX + "delegation.token.max-lifetime";
677 public static final long RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
678 7*24*60*60*1000; // 7 days
679
680 public static final String RM_DELEGATION_TOKEN_MAX_CONF_SIZE =
681 RM_PREFIX + "delegation-token.max-conf-size-bytes";
682 public static final int DEFAULT_RM_DELEGATION_TOKEN_MAX_CONF_SIZE_BYTES =
683 12800;
684
685 public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled";
686 public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false;
687
688 public static final String YARN_FAIL_FAST = YARN_PREFIX + "fail-fast";
689 public static final boolean DEFAULT_YARN_FAIL_FAST = false;
690
691 public static final String RM_FAIL_FAST = RM_PREFIX + "fail-fast";
692
693 @Private
694 public static final String RM_WORK_PRESERVING_RECOVERY_ENABLED = RM_PREFIX
695 + "work-preserving-recovery.enabled";
696 @Private
697 public static final boolean DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED =
698 true;
699
700 public static final String RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS =
701 RM_PREFIX + "work-preserving-recovery.scheduling-wait-ms";
702 public static final long DEFAULT_RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS =
703 10000;
704
705 /** Zookeeper interaction configs */
706 public static final String RM_ZK_PREFIX = RM_PREFIX + "zk-";
707
708 public static final String RM_ZK_ADDRESS = RM_ZK_PREFIX + "address";
709
710 public static final String RM_ZK_NUM_RETRIES = RM_ZK_PREFIX + "num-retries";
711 public static final int DEFAULT_ZK_RM_NUM_RETRIES = 1000;
712
713 /** Zookeeper znode limit */
714 public static final String RM_ZK_ZNODE_SIZE_LIMIT_BYTES =
715 RM_ZK_PREFIX + "max-znode-size.bytes";
716 public static final int DEFAULT_RM_ZK_ZNODE_SIZE_LIMIT_BYTES = 1024 * 1024;
717
718 public static final String RM_ZK_RETRY_INTERVAL_MS =
719 RM_ZK_PREFIX + "retry-interval-ms";
720 public static final int DEFAULT_RM_ZK_RETRY_INTERVAL_MS = 1000;
721
722 public static final String RM_ZK_TIMEOUT_MS = RM_ZK_PREFIX + "timeout-ms";
723 public static final int DEFAULT_RM_ZK_TIMEOUT_MS = 10000;
724
725 public static final String ZK_APPID_NODE_SPLIT_INDEX =
726 RM_ZK_PREFIX + "appid-node.split-index";
727 public static final int DEFAULT_ZK_APPID_NODE_SPLIT_INDEX = 0;
728
729 /** Index at which the RM Delegation Token ids will be split so that the
730 * delegation token znodes stored in the zookeeper RM state store will be
731 * stored as two different znodes (parent-child). **/
732 public static final String ZK_DELEGATION_TOKEN_NODE_SPLIT_INDEX =
733 RM_ZK_PREFIX + "delegation-token-node.split-index";
734 public static final int DEFAULT_ZK_DELEGATION_TOKEN_NODE_SPLIT_INDEX = 0;
735
736 public static final String RM_ZK_ACL = RM_ZK_PREFIX + "acl";
737 public static final String DEFAULT_RM_ZK_ACL = "world:anyone:rwcda";
738
739 public static final String RM_ZK_AUTH = RM_ZK_PREFIX + "auth";
740
741 public static final String ZK_STATE_STORE_PREFIX =
742 RM_PREFIX + "zk-state-store.";
743
744 /** Parent znode path under which ZKRMStateStore will create znodes */
745 public static final String ZK_RM_STATE_STORE_PARENT_PATH =
746 ZK_STATE_STORE_PREFIX + "parent-path";
747 public static final String DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH = "/rmstore";
748
749 /** Root node ACLs for fencing */
750 public static final String ZK_RM_STATE_STORE_ROOT_NODE_ACL =
751 ZK_STATE_STORE_PREFIX + "root-node.acl";
752
753 /** HA related configs */
754 public static final String RM_HA_PREFIX = RM_PREFIX + "ha.";
755 public static final String RM_HA_ENABLED = RM_HA_PREFIX + "enabled";
756 public static final boolean DEFAULT_RM_HA_ENABLED = false;
757
758 public static final String RM_HA_IDS = RM_HA_PREFIX + "rm-ids";
759 public static final String RM_HA_ID = RM_HA_PREFIX + "id";
760
761 /** Store the related configuration files in File System */
762 public static final String FS_BASED_RM_CONF_STORE = RM_PREFIX
763 + "configuration.file-system-based-store";
764 public static final String DEFAULT_FS_BASED_RM_CONF_STORE = "/yarn/conf";
765
766 public static final String RM_CONFIGURATION_PROVIDER_CLASS = RM_PREFIX
767 + "configuration.provider-class";
768 public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS =
769 "org.apache.hadoop.yarn.LocalConfigurationProvider";
770
771 @Private
772 @Unstable
773 public static final String SCHEDULER_CONFIGURATION_STORE_CLASS =
774 YARN_PREFIX + "scheduler.configuration.store.class";
775 @Private
776 @Unstable
777 public static final String FILE_CONFIGURATION_STORE = "file";
778 @Private
779 @Unstable
780 public static final String MEMORY_CONFIGURATION_STORE = "memory";
781 @Private
782 @Unstable
783 public static final String FS_CONFIGURATION_STORE = "fs";
784 @Private
785 @Unstable
786 public static final String ZK_CONFIGURATION_STORE = "zk";
787 @Private
788 @Unstable
789 public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
790 @Private
791 @Unstable
792 public static final String DEFAULT_CONFIGURATION_STORE =
793 FILE_CONFIGURATION_STORE;
794 @Private
795 @Unstable
796 public static final String RM_SCHEDCONF_STORE_PATH = YARN_PREFIX
797 + "scheduler.configuration.leveldb-store.path";
798
799 @Private
800 @Unstable
801 public static final String RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS =
802 YARN_PREFIX
803 + "scheduler.configuration.leveldb-store.compaction-interval-secs";
804 @Private
805 @Unstable
806 public static final long
807 DEFAULT_RM_SCHEDCONF_LEVELDB_COMPACTION_INTERVAL_SECS = 60 * 60 * 24L;
808
809 @Private
810 @Unstable
811 public static final String RM_SCHEDCONF_MAX_LOGS =
812 YARN_PREFIX + "scheduler.configuration.store.max-logs";
813 @Private
814 @Unstable
815 public static final long DEFAULT_RM_SCHEDCONF_LEVELDB_MAX_LOGS = 1000;
816 @Private
817 @Unstable
818 public static final long DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS = 1000;
819 @Private
820 @Unstable
821 public static final String SCHEDULER_CONFIGURATION_FS_PATH =
822 YARN_PREFIX + "scheduler.configuration.fs.path";
823 @Private
824 @Unstable
825 public static final String SCHEDULER_CONFIGURATION_FS_MAX_VERSION =
826 YARN_PREFIX + "scheduler.configuration.max.version";
827 @Private
828 @Unstable
829 public static final int DEFAULT_SCHEDULER_CONFIGURATION_FS_MAX_VERSION = 100;
830
831 /** Parent znode path under which ZKConfigurationStore will create znodes. */
832 @Private
833 @Unstable
834 public static final String RM_SCHEDCONF_STORE_ZK_PARENT_PATH = YARN_PREFIX
835 + "scheduler.configuration.zk-store.parent-path";
836 @Private
837 @Unstable
838 public static final String DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH =
839 "/confstore";
840
841 @Private
842 @Unstable
843 public static final String RM_SCHEDULER_MUTATION_ACL_POLICY_CLASS =
844 YARN_PREFIX + "scheduler.configuration.mutation.acl-policy.class";
845
846 public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX
847 + "authorization-provider";
848 private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTP =
849 Collections.unmodifiableList(Arrays.asList(
850 RM_ADDRESS,
851 RM_SCHEDULER_ADDRESS,
852 RM_ADMIN_ADDRESS,
853 RM_RESOURCE_TRACKER_ADDRESS,
854 RM_WEBAPP_ADDRESS));
855
856 private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTPS =
857 Collections.unmodifiableList(Arrays.asList(
858 RM_ADDRESS,
859 RM_SCHEDULER_ADDRESS,
860 RM_ADMIN_ADDRESS,
861 RM_RESOURCE_TRACKER_ADDRESS,
862 RM_WEBAPP_HTTPS_ADDRESS));
863
864 public static final String AUTO_FAILOVER_PREFIX =
865 RM_HA_PREFIX + "automatic-failover.";
866
867 public static final String AUTO_FAILOVER_ENABLED =
868 AUTO_FAILOVER_PREFIX + "enabled";
869 public static final boolean DEFAULT_AUTO_FAILOVER_ENABLED = true;
870
871 /**
872 * This property controls whether {@link ActiveStandbyElector} leader
873 * election should be used when {@link #CURATOR_LEADER_ELECTOR} is
874 * {@code false}.
875 *
876 * @deprecated This property should never be set to {@code false}.
877 */
878 @Deprecated
879 public static final String AUTO_FAILOVER_EMBEDDED =
880 AUTO_FAILOVER_PREFIX + "embedded";
881 /**
882 * The default value for {@link #AUTO_FAILOVER_EMBEDDED}.
883 *
884 * @deprecated The {@link #AUTO_FAILOVER_EMBEDDED} property is deprecated.
885 */
886 @Deprecated
887 public static final boolean DEFAULT_AUTO_FAILOVER_EMBEDDED = true;
888
889 public static final String AUTO_FAILOVER_ZK_BASE_PATH =
890 AUTO_FAILOVER_PREFIX + "zk-base-path";
891 public static final String DEFAULT_AUTO_FAILOVER_ZK_BASE_PATH =
892 "/yarn-leader-election";
893
894 public static final String CLIENT_FAILOVER_PREFIX =
895 YARN_PREFIX + "client.failover-";
896 public static final String CLIENT_FAILOVER_PROXY_PROVIDER =
897 CLIENT_FAILOVER_PREFIX + "proxy-provider";
898 public static final String DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER =
899 "org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider";
900
901 public static final String CLIENT_FAILOVER_MAX_ATTEMPTS =
902 CLIENT_FAILOVER_PREFIX + "max-attempts";
903
904 public static final String CLIENT_FAILOVER_SLEEPTIME_BASE_MS =
905 CLIENT_FAILOVER_PREFIX + "sleep-base-ms";
906
907 public static final String CLIENT_FAILOVER_SLEEPTIME_MAX_MS =
908 CLIENT_FAILOVER_PREFIX + "sleep-max-ms";
909
910 public static final String CLIENT_FAILOVER_RETRIES =
911 CLIENT_FAILOVER_PREFIX + "retries";
912 public static final int DEFAULT_CLIENT_FAILOVER_RETRIES = 0;
913
914 public static final String CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS =
915 CLIENT_FAILOVER_PREFIX + "retries-on-socket-timeouts";
916 public static final int
917 DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS = 0;
918
919 /** number of zookeeper operation retry times in ActiveStandbyElector */
920 public static final String RM_HA_FC_ELECTOR_ZK_RETRIES_KEY = RM_HA_PREFIX
921 + "failover-controller.active-standby-elector.zk.retries";
922
923
924 /**
925 * Whether to use the Curator-based elector for leader election.
926 *
927 * @deprecated Eventually, we want to default to the curator-based
928 * implementation and remove the {@link ActiveStandbyElector} based
929 * implementation. We should remove this config then.
930 */
931 @Unstable
932 @Deprecated
933 public static final String CURATOR_LEADER_ELECTOR =
934 RM_HA_PREFIX + "curator-leader-elector.enabled";
935 @Private
936 @Unstable
937 public static final boolean DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED = false;
938
939 ////////////////////////////////
940 // RM state store configs
941 ////////////////////////////////
942 /** The class to use as the persistent store.*/
943 public static final String RM_STORE = RM_PREFIX + "store.class";
944
945 /** URI for FileSystemRMStateStore */
946 public static final String FS_RM_STATE_STORE_URI = RM_PREFIX
947 + "fs.state-store.uri";
948
949 public static final String FS_RM_STATE_STORE_NUM_RETRIES =
950 RM_PREFIX + "fs.state-store.num-retries";
951 public static final int DEFAULT_FS_RM_STATE_STORE_NUM_RETRIES = 0;
952
953 public static final String FS_RM_STATE_STORE_RETRY_INTERVAL_MS =
954 RM_PREFIX + "fs.state-store.retry-interval-ms";
955 public static final long DEFAULT_FS_RM_STATE_STORE_RETRY_INTERVAL_MS =
956 1000L;
957
958 public static final String RM_LEVELDB_STORE_PATH = RM_PREFIX
959 + "leveldb-state-store.path";
960
961 /** The time in seconds between full compactions of the leveldb database.
962 * Setting the interval to zero disables the full compaction cycles.
963 */
964 public static final String RM_LEVELDB_COMPACTION_INTERVAL_SECS = RM_PREFIX
965 + "leveldb-state-store.compaction-interval-secs";
966 public static final long DEFAULT_RM_LEVELDB_COMPACTION_INTERVAL_SECS = 3600;
967
968 /**
969 * The maximum number of completed applications RM keeps. By default equals
970 * to {@link #DEFAULT_RM_MAX_COMPLETED_APPLICATIONS}.
971 */
972 public static final String RM_MAX_COMPLETED_APPLICATIONS =
973 RM_PREFIX + "max-completed-applications";
974 public static final int DEFAULT_RM_MAX_COMPLETED_APPLICATIONS = 1000;
975
976 /**
977 * The maximum number of completed applications RM state store keeps. By
978 * default equals to value of {@link #RM_MAX_COMPLETED_APPLICATIONS}.
979 */
980 public static final String RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS =
981 RM_PREFIX + "state-store.max-completed-applications";
982 /**
983 * The default value for
984 * {@code yarn.resourcemanager.state-store.max-completed-applications}.
985 * @deprecated This default value is ignored and will be removed in a future
986 * release. The default value of
987 * {@code yarn.resourcemanager.state-store.max-completed-applications} is the
988 * value of {@link #RM_MAX_COMPLETED_APPLICATIONS}.
989 */
990 @Deprecated
991 public static final int DEFAULT_RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS =
992 DEFAULT_RM_MAX_COMPLETED_APPLICATIONS;
993
994 /** Default application name */
995 public static final String DEFAULT_APPLICATION_NAME = "N/A";
996
997 /** Default application type */
998 public static final String DEFAULT_APPLICATION_TYPE = "YARN";
999
1000 /** Default application type length */
1001 public static final int APPLICATION_TYPE_LENGTH = 20;
1002
1003 /** Default queue name */
1004 public static final String DEFAULT_QUEUE_NAME = "default";
1005
1006 /**
1007 * Buckets (in minutes) for the number of apps running in each queue.
1008 */
1009 public static final String RM_METRICS_RUNTIME_BUCKETS =
1010 RM_PREFIX + "metrics.runtime.buckets";
1011
1012 /**
1013 * Default sizes of the runtime metric buckets in minutes.
1014 */
1015 public static final String DEFAULT_RM_METRICS_RUNTIME_BUCKETS =
1016 "60,300,1440";
1017
1018 public static final String RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX
1019 + "am-rm-tokens.master-key-rolling-interval-secs";
1020
1021 public static final long DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
1022 24 * 60 * 60;
1023
1024 public static final String RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
1025 RM_PREFIX + "container-tokens.master-key-rolling-interval-secs";
1026
1027 public static final long DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
1028 24 * 60 * 60;
1029
1030 public static final String RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
1031 RM_PREFIX + "nm-tokens.master-key-rolling-interval-secs";
1032
1033 public static final long DEFAULT_RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
1034 24 * 60 * 60;
1035
1036 public static final String RM_NODEMANAGER_MINIMUM_VERSION =
1037 RM_PREFIX + "nodemanager.minimum.version";
1038
1039 public static final String DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION =
1040 "NONE";
1041
1042 /**
1043 * Timeout(msec) for an untracked node to remain in shutdown or decommissioned
1044 * state.
1045 */
1046 public static final String RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC =
1047 RM_PREFIX + "node-removal-untracked.timeout-ms";
1048 public static final int
1049 DEFAULT_RM_NODEMANAGER_UNTRACKED_REMOVAL_TIMEOUT_MSEC = 60000;
1050
1051 /**
1052 * RM proxy users' prefix
1053 */
1054 public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser.";
1055
1056 /**
1057 * Enable/disable resource profiles.
1058 */
1059 @Public
1060 @Unstable
1061 public static final String RM_RESOURCE_PROFILES_ENABLED =
1062 RM_PREFIX + "resource-profiles.enabled";
1063 @Public
1064 @Unstable
1065 public static final boolean DEFAULT_RM_RESOURCE_PROFILES_ENABLED = false;
1066
1067 /**
1068 * File containing resource profiles.
1069 */
1070 @Public
1071 @Unstable
1072 public static final String RM_RESOURCE_PROFILES_SOURCE_FILE =
1073 RM_PREFIX + "resource-profiles.source-file";
1074 @Public
1075 @Unstable
1076 public static final String DEFAULT_RM_RESOURCE_PROFILES_SOURCE_FILE =
1077 "resource-profiles.json";
1078
1079 /**
1080 * Enable/disable loading resource-types.xml at client side.
1081 */
1082 @Public
1083 @Unstable
1084 public static final String YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER = YARN_PREFIX
1085 + "client.load.resource-types.from-server";
1086 @Public
1087 @Unstable
1088 public static final boolean DEFAULT_YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER = false;
1089
1090 /**
1091 * Timeout in seconds for YARN node graceful decommission.
1092 * This is the maximal time to wait for running containers and applications
1093 * to complete before transition a DECOMMISSIONING node into DECOMMISSIONED.
1094 */
1095 public static final String RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT =
1096 RM_PREFIX + "nodemanager-graceful-decommission-timeout-secs";
1097 public static final int DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT = 3600;
1098
1099 /**
1100 * Period in seconds of the poll timer task inside DecommissioningNodesWatcher
1101 * to identify and take care of DECOMMISSIONING nodes missing regular heart beat.
1102 */
1103 public static final String RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL =
1104 RM_PREFIX + "decommissioning-nodes-watcher.poll-interval-secs";
1105 public static final int
1106 DEFAULT_RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL = 20;
1107
1108 ////////////////////////////////
1109 // Node Manager Configs
1110 ////////////////////////////////
1111
1112 /** Prefix for all node manager configs.*/
1113 public static final String NM_PREFIX = "yarn.nodemanager.";
1114
1115 /** Max Queue length of <code>OPPORTUNISTIC</code> containers on the NM. */
1116 public static final String NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH =
1117 NM_PREFIX + "opportunistic-containers-max-queue-length";
1118 public static final int DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH =
1119 0;
1120
1121 /** Setting that controls whether distributed scheduling is enabled or not. */
1122 public static final String DIST_SCHEDULING_ENABLED =
1123 NM_PREFIX + "distributed-scheduling.enabled";
1124 public static final boolean DEFAULT_DIST_SCHEDULING_ENABLED = false;
1125
1126 /** Environment variables that will be sent to containers.*/
1127 public static final String NM_ADMIN_USER_ENV = NM_PREFIX + "admin-env";
1128 public static final String DEFAULT_NM_ADMIN_USER_ENV = "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX";
1129
1130 /** Environment variables that containers may override rather than use NodeManager's default.*/
1131 public static final String NM_ENV_WHITELIST = NM_PREFIX + "env-whitelist";
1132 public static final String DEFAULT_NM_ENV_WHITELIST = StringUtils.join(",",
1133 Arrays.asList(ApplicationConstants.Environment.JAVA_HOME.key(),
1134 ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(),
1135 ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(),
1136 ApplicationConstants.Environment.HADOOP_CONF_DIR.key(),
1137 ApplicationConstants.Environment.CLASSPATH_PREPEND_DISTCACHE.key(),
1138 ApplicationConstants.Environment.HADOOP_YARN_HOME.key()));
1139
1140 /** address of node manager IPC.*/
1141 public static final String NM_ADDRESS = NM_PREFIX + "address";
1142 public static final int DEFAULT_NM_PORT = 0;
1143 public static final String DEFAULT_NM_ADDRESS = "0.0.0.0:"
1144 + DEFAULT_NM_PORT;
1145
1146 /** The actual bind address or the NM.*/
1147 public static final String NM_BIND_HOST =
1148 NM_PREFIX + "bind-host";
1149
1150 /** who will execute(launch) the containers.*/
1151 public static final String NM_CONTAINER_EXECUTOR =
1152 NM_PREFIX + "container-executor.class";
1153
1154 /** List of container state transition listeners.*/
1155 public static final String NM_CONTAINER_STATE_TRANSITION_LISTENERS =
1156 NM_PREFIX + "container-state-transition-listener.classes";
1157
1158 /**
1159 * Adjustment to make to the container os scheduling priority.
1160 * The valid values for this could vary depending on the platform.
1161 * On Linux, higher values mean run the containers at a less
1162 * favorable priority than the NM.
1163 * The value specified is an int.
1164 */
1165 public static final String NM_CONTAINER_EXECUTOR_SCHED_PRIORITY =
1166 NM_PREFIX + "container-executor.os.sched.priority.adjustment";
1167 public static final int DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 0;
1168
1169 /** Number of threads container manager uses.*/
1170 public static final String NM_CONTAINER_MGR_THREAD_COUNT =
1171 NM_PREFIX + "container-manager.thread-count";
1172 public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20;
1173
1174 /** Number of threads container manager uses.*/
1175 public static final String NM_COLLECTOR_SERVICE_THREAD_COUNT =
1176 NM_PREFIX + "collector-service.thread-count";
1177 public static final int DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT = 5;
1178
1179 /** Number of threads used in cleanup.*/
1180 public static final String NM_DELETE_THREAD_COUNT =
1181 NM_PREFIX + "delete.thread-count";
1182 public static final int DEFAULT_NM_DELETE_THREAD_COUNT = 4;
1183
1184 /** Keytab for NM.*/
1185 public static final String NM_KEYTAB = NM_PREFIX + "keytab";
1186
1187 /**List of directories to store localized files in.*/
1188 public static final String NM_LOCAL_DIRS = NM_PREFIX + "local-dirs";
1189 public static final String DEFAULT_NM_LOCAL_DIRS = "/tmp/nm-local-dir";
1190
1191 /**
1192 * Number of files in each localized directories
1193 * Avoid tuning this too low.
1194 */
1195 public static final String NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY =
1196 NM_PREFIX + "local-cache.max-files-per-directory";
1197 public static final int DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = 8192;
1198
1199 /** Address where the localizer IPC is.*/
1200 public static final String NM_LOCALIZER_ADDRESS =
1201 NM_PREFIX + "localizer.address";
1202 public static final int DEFAULT_NM_LOCALIZER_PORT = 8040;
1203 public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" +
1204 DEFAULT_NM_LOCALIZER_PORT;
1205
1206 /** Address where the collector service IPC is.*/
1207 public static final String NM_COLLECTOR_SERVICE_ADDRESS =
1208 NM_PREFIX + "collector-service.address";
1209 public static final int DEFAULT_NM_COLLECTOR_SERVICE_PORT = 8048;
1210 public static final String DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS =
1211 "0.0.0.0:" + DEFAULT_NM_LOCALIZER_PORT;
1212
1213 /** Interval in between cache cleanups.*/
1214 public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS =
1215 NM_PREFIX + "localizer.cache.cleanup.interval-ms";
1216 public static final long DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS =
1217 10 * 60 * 1000;
1218
1219 /**
1220 * Target size of localizer cache in MB, per nodemanager. It is a target
1221 * retention size that only includes resources with PUBLIC and PRIVATE
1222 * visibility and excludes resources with APPLICATION visibility
1223 */
1224 public static final String NM_LOCALIZER_CACHE_TARGET_SIZE_MB =
1225 NM_PREFIX + "localizer.cache.target-size-mb";
1226 public static final long DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 10 * 1024;
1227
1228 /** Number of threads to handle localization requests.*/
1229 public static final String NM_LOCALIZER_CLIENT_THREAD_COUNT =
1230 NM_PREFIX + "localizer.client.thread-count";
1231 public static final int DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT = 5;
1232
1233 /** Number of threads to use for localization fetching.*/
1234 public static final String NM_LOCALIZER_FETCH_THREAD_COUNT =
1235 NM_PREFIX + "localizer.fetch.thread-count";
1236 public static final int DEFAULT_NM_LOCALIZER_FETCH_THREAD_COUNT = 4;
1237
1238 /** Where to store container logs.*/
1239 public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs";
1240 public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs";
1241
1242 /** The number of threads to handle log aggregation in node manager. */
1243 public static final String NM_LOG_AGGREGATION_THREAD_POOL_SIZE =
1244 NM_PREFIX + "logaggregation.threadpool-size-max";
1245 public static final int DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE = 100;
1246
1247 /** Default permissions for container logs. */
1248 public static final String NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX =
1249 NM_PREFIX + "default-container-executor.";
1250 public static final String NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS =
1251 NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX + "log-dirs.permissions";
1252 public static final String NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS_DEFAULT = "710";
1253
1254 public static final String NM_RESOURCEMANAGER_MINIMUM_VERSION =
1255 NM_PREFIX + "resourcemanager.minimum.version";
1256 public static final String DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION = "NONE";
1257
1258 /** Disk Validator. */
1259 public static final String DISK_VALIDATOR = NM_PREFIX + "disk-validator";
1260 public static final String DEFAULT_DISK_VALIDATOR = BasicDiskValidator.NAME;
1261
1262 /**
1263 * Maximum size of contain's diagnostics to keep for relaunching container
1264 * case.
1265 **/
1266 public static final String NM_CONTAINER_DIAGNOSTICS_MAXIMUM_SIZE =
1267 NM_PREFIX + "container-diagnostics-maximum-size";
1268 public static final int DEFAULT_NM_CONTAINER_DIAGNOSTICS_MAXIMUM_SIZE = 10000;
1269
1270 /** Minimum container restart interval. */
1271 public static final String NM_CONTAINER_RETRY_MINIMUM_INTERVAL_MS =
1272 NM_PREFIX + "container-retry-minimum-interval-ms";
1273 public static final int DEFAULT_NM_CONTAINER_RETRY_MINIMUM_INTERVAL_MS = 1000;
1274
1275 /**
1276 * Use container pause as the preemption policy over kill in the container
1277 * queue at a NodeManager.
1278 **/
1279 public static final String NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION =
1280 NM_PREFIX + "opportunistic-containers-use-pause-for-preemption";
1281 public static final boolean
1282 DEFAULT_NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION = false;
1283
1284 /** Interval at which the delayed token removal thread runs */
1285 public static final String RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS =
1286 RM_PREFIX + "delayed.delegation-token.removal-interval-ms";
1287 public static final long DEFAULT_RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS =
1288 30000l;
1289
1290 /** Delegation Token renewer thread count */
1291 public static final String RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT =
1292 RM_PREFIX + "delegation-token-renewer.thread-count";
1293 public static final int DEFAULT_RM_DELEGATION_TOKEN_RENEWER_THREAD_COUNT = 50;
1294
1295 public static final String RM_PROXY_USER_PRIVILEGES_ENABLED = RM_PREFIX
1296 + "proxy-user-privileges.enabled";
1297 public static final boolean DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED = false;
1298
1299 /** The expiry interval for node IP caching. -1 disables the caching */
1300 public static final String RM_NODE_IP_CACHE_EXPIRY_INTERVAL_SECS = RM_PREFIX
1301 + "node-ip-cache.expiry-interval-secs";
1302 public static final int DEFAULT_RM_NODE_IP_CACHE_EXPIRY_INTERVAL_SECS = -1;
1303
1304 /**
1305 * How many diagnostics/failure messages can be saved in RM for
1306 * log aggregation. It also defines the number of diagnostics/failure
1307 * messages can be shown in log aggregation web ui.
1308 */
1309 public static final String RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY =
1310 RM_PREFIX + "max-log-aggregation-diagnostics-in-memory";
1311 public static final int DEFAULT_RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY =
1312 10;
1313
1314 /** Whether to enable log aggregation */
1315 public static final String LOG_AGGREGATION_ENABLED = YARN_PREFIX
1316 + "log-aggregation-enable";
1317 public static final boolean DEFAULT_LOG_AGGREGATION_ENABLED = false;
1318
1319 public static final String LOG_AGGREGATION_FILE_FORMATS = YARN_PREFIX
1320 + "log-aggregation.file-formats";
1321 public static final String LOG_AGGREGATION_FILE_CONTROLLER_FMT =
1322 YARN_PREFIX + "log-aggregation.file-controller.%s.class";
1323
1324 public static final String LOG_AGGREGATION_REMOTE_APP_LOG_DIR_FMT
1325 = YARN_PREFIX + "log-aggregation.%s.remote-app-log-dir";
1326 public static final String LOG_AGGREGATION_REMOTE_APP_LOG_DIR_SUFFIX_FMT
1327 = YARN_PREFIX + "log-aggregation.%s.remote-app-log-dir-suffix";
1328
1329 /**
1330 * How long to wait before deleting aggregated logs, -1 disables.
1331 * Be careful set this too small and you will spam the name node.
1332 */
1333 public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX
1334 + "log-aggregation.retain-seconds";
1335 public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1;
1336
1337 /**
1338 * How long to wait between aggregated log retention checks. If set to
1339 * a value {@literal <=} 0 then the value is computed as one-tenth of the
1340 * log retention setting. Be careful set this too small and you will spam
1341 * the name node.
1342 */
1343 public static final String LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS =
1344 YARN_PREFIX + "log-aggregation.retain-check-interval-seconds";
1345 public static final long DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = -1;
1346
1347 /**
1348 * How long for ResourceManager to wait for NodeManager to report its
1349 * log aggregation status. If waiting time of which the log aggregation status
1350 * is reported from NodeManager exceeds the configured value, RM will report
1351 * log aggregation status for this NodeManager as TIME_OUT.
1352 *
1353 * This configuration will be used in NodeManager as well to decide
1354 * whether and when to delete the cached log aggregation status.
1355 */
1356 public static final String LOG_AGGREGATION_STATUS_TIME_OUT_MS =
1357 YARN_PREFIX + "log-aggregation-status.time-out.ms";
1358 public static final long DEFAULT_LOG_AGGREGATION_STATUS_TIME_OUT_MS
1359 = 10 * 60 * 1000;
1360
1361 /**
1362 * Number of seconds to retain logs on the NodeManager. Only applicable if Log
1363 * aggregation is disabled
1364 */
1365 public static final String NM_LOG_RETAIN_SECONDS = NM_PREFIX
1366 + "log.retain-seconds";
1367 public static final long DEFAULT_NM_LOG_RETAIN_SECONDS = 3 * 60 * 60;
1368
1369 /**
1370 * Define how often NMs wake up and upload log files
1371 */
1372 public static final String NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS =
1373 NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds";
1374 public static final long
1375 DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1;
1376 /**
1377 * Number of threads used in log cleanup. Only applicable if Log aggregation
1378 * is disabled
1379 */
1380 public static final String NM_LOG_DELETION_THREADS_COUNT =
1381 NM_PREFIX + "log.deletion-threads-count";
1382 public static final int DEFAULT_NM_LOG_DELETE_THREAD_COUNT = 4;
1383
1384 /** Where to aggregate logs to.*/
1385 public static final String NM_REMOTE_APP_LOG_DIR =
1386 NM_PREFIX + "remote-app-log-dir";
1387 public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR = "/tmp/logs";
1388
1389 /**
1390 * The remote log dir will be created at
1391 * NM_REMOTE_APP_LOG_DIR/${user}/NM_REMOTE_APP_LOG_DIR_SUFFIX/${appId}
1392 */
1393 public static final String NM_REMOTE_APP_LOG_DIR_SUFFIX =
1394 NM_PREFIX + "remote-app-log-dir-suffix";
1395 public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX="logs";
1396
1397 public static final String YARN_LOG_SERVER_URL =
1398 YARN_PREFIX + "log.server.url";
1399
1400 public static final String YARN_LOG_SERVER_WEBSERVICE_URL =
1401 YARN_PREFIX + "log.server.web-service.url";
1402
1403 public static final String YARN_TRACKING_URL_GENERATOR =
1404 YARN_PREFIX + "tracking.url.generator";
1405
1406 /** Amount of memory in MB that can be allocated for containers.*/
1407 public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb";
1408 public static final int DEFAULT_NM_PMEM_MB = 8 * 1024;
1409
1410 /** Amount of memory in MB that has been reserved for non-yarn use. */
1411 public static final String NM_SYSTEM_RESERVED_PMEM_MB = NM_PREFIX
1412 + "resource.system-reserved-memory-mb";
1413
1414 /** Specifies whether physical memory check is enabled. */
1415 public static final String NM_PMEM_CHECK_ENABLED = NM_PREFIX
1416 + "pmem-check-enabled";
1417 public static final boolean DEFAULT_NM_PMEM_CHECK_ENABLED = true;
1418
1419 /** Specifies whether physical memory check is enabled. */
1420 public static final String NM_VMEM_CHECK_ENABLED = NM_PREFIX
1421 + "vmem-check-enabled";
1422 public static final boolean DEFAULT_NM_VMEM_CHECK_ENABLED = true;
1423
1424 /** Conversion ratio for physical memory to virtual memory. */
1425 public static final String NM_VMEM_PMEM_RATIO =
1426 NM_PREFIX + "vmem-pmem-ratio";
1427 public static final float DEFAULT_NM_VMEM_PMEM_RATIO = 2.1f;
1428
1429 /** Number of Virtual CPU Cores which can be allocated for containers.*/
1430 public static final String NM_VCORES = NM_PREFIX + "resource.cpu-vcores";
1431 public static final int DEFAULT_NM_VCORES = 8;
1432
1433 /** Count logical processors(like hyperthreads) as cores. */
1434 public static final String NM_COUNT_LOGICAL_PROCESSORS_AS_CORES = NM_PREFIX
1435 + "resource.count-logical-processors-as-cores";
1436 public static final boolean DEFAULT_NM_COUNT_LOGICAL_PROCESSORS_AS_CORES =
1437 false;
1438
1439 /** Multiplier to convert physical cores to vcores. */
1440 public static final String NM_PCORES_VCORES_MULTIPLIER = NM_PREFIX
1441 + "resource.pcores-vcores-multiplier";
1442 public static final float DEFAULT_NM_PCORES_VCORES_MULTIPLIER = 1.0f;
1443
1444 /** Percentage of overall CPU which can be allocated for containers. */
1445 public static final String NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT =
1446 NM_PREFIX + "resource.percentage-physical-cpu-limit";
1447 public static final int DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT =
1448 100;
1449
1450 /** Enable or disable node hardware capability detection. */
1451 public static final String NM_ENABLE_HARDWARE_CAPABILITY_DETECTION =
1452 NM_PREFIX + "resource.detect-hardware-capabilities";
1453 public static final boolean DEFAULT_NM_ENABLE_HARDWARE_CAPABILITY_DETECTION =
1454 false;
1455
1456 @Private
1457 public static final String NM_MEMORY_RESOURCE_PREFIX = NM_PREFIX
1458 + "resource.memory.";
1459
1460 public static final String NM_MEMORY_RESOURCE_ENABLED =
1461 NM_MEMORY_RESOURCE_PREFIX + "enabled";
1462 public static final boolean DEFAULT_NM_MEMORY_RESOURCE_ENABLED = false;
1463
1464 public static final String NM_MEMORY_RESOURCE_ENFORCED =
1465 NM_MEMORY_RESOURCE_PREFIX + "enforced";
1466 public static final boolean DEFAULT_NM_MEMORY_RESOURCE_ENFORCED = true;
1467
1468 public static final String NM_MEMORY_RESOURCE_CGROUPS_SWAPPINESS =
1469 NM_MEMORY_RESOURCE_PREFIX + "cgroups.swappiness";
1470 public static final int DEFAULT_NM_MEMORY_RESOURCE_CGROUPS_SWAPPINESS = 0;
1471
1472 public static final String NM_MEMORY_RESOURCE_CGROUPS_SOFT_LIMIT_PERCENTAGE =
1473 NM_MEMORY_RESOURCE_PREFIX + "cgroups.soft-limit-percentage";
1474 public static final float
1475 DEFAULT_NM_MEMORY_RESOURCE_CGROUPS_SOFT_LIMIT_PERCENTAGE =
1476 90.0f;
1477
1478 @Private
1479 public static final String NM_CPU_RESOURCE_PREFIX = NM_PREFIX
1480 + "resource.cpu.";
1481
1482 /** Enable cpu isolation. */
1483 @Private
1484 public static final String NM_CPU_RESOURCE_ENABLED =
1485 NM_CPU_RESOURCE_PREFIX + "enabled";
1486
1487 @Private
1488 public static final boolean DEFAULT_NM_CPU_RESOURCE_ENABLED = false;
1489
1490 /**
1491 * Prefix for disk configurations. Work in progress: This configuration
1492 * parameter may be changed/removed in the future.
1493 */
1494 @Private
1495 public static final String NM_DISK_RESOURCE_PREFIX = NM_PREFIX
1496 + "resource.disk.";
1497 /**
1498 * This setting controls if resource handling for disk operations is enabled.
1499 * Work in progress: This configuration parameter may be changed/removed in
1500 * the future
1501 */
1502 @Private
1503 public static final String NM_DISK_RESOURCE_ENABLED = NM_DISK_RESOURCE_PREFIX
1504 + "enabled";
1505 /** Disk as a resource is disabled by default. **/
1506 @Private
1507 public static final boolean DEFAULT_NM_DISK_RESOURCE_ENABLED = false;
1508
1509 public static final String NM_NETWORK_RESOURCE_PREFIX = NM_PREFIX
1510 + "resource.network.";
1511
1512 /**
1513 * This setting controls if resource handling for network bandwidth is
1514 * enabled. Work in progress: This configuration parameter may be
1515 * changed/removed in the future
1516 */
1517 @Private
1518 public static final String NM_NETWORK_RESOURCE_ENABLED =
1519 NM_NETWORK_RESOURCE_PREFIX + "enabled";
1520 /** Network as a resource is disabled by default. **/
1521 @Private
1522 public static final boolean DEFAULT_NM_NETWORK_RESOURCE_ENABLED = false;
1523
1524 /**
1525 * Specifies the interface to be used for applying network throttling rules.
1526 * Work in progress: This configuration parameter may be changed/removed in
1527 * the future
1528 */
1529 @Private
1530 public static final String NM_NETWORK_RESOURCE_INTERFACE =
1531 NM_NETWORK_RESOURCE_PREFIX + "interface";
1532 @Private
1533 public static final String DEFAULT_NM_NETWORK_RESOURCE_INTERFACE = "eth0";
1534
1535 /**
1536 * Specifies the total available outbound bandwidth on the node. Work in
1537 * progress: This configuration parameter may be changed/removed in the future
1538 */
1539 @Private
1540 public static final String NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT =
1541 NM_NETWORK_RESOURCE_PREFIX + "outbound-bandwidth-mbit";
1542 @Private
1543 public static final int DEFAULT_NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT =
1544 1000;
1545
1546 /**
1547 * Specifies the total outbound bandwidth available to YARN containers.
1548 * defaults to NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT if not specified.
1549 * Work in progress: This configuration parameter may be changed/removed in
1550 * the future
1551 */
1552 @Private
1553 public static final String NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_YARN_MBIT =
1554 NM_NETWORK_RESOURCE_PREFIX + "outbound-bandwidth-yarn-mbit";
1555
1556 /**
1557 * Prefix for computation resources, example of computation resources like
1558 * GPU / FPGA / TPU, etc.
1559 */
1560 @Private
1561 public static final String NM_RESOURCE_PLUGINS =
1562 NM_PREFIX + "resource-plugins";
1563
1564 /**
1565 * Prefix for gpu configurations. Work in progress: This configuration
1566 * parameter may be changed/removed in the future.
1567 */
1568 @Private
1569 public static final String NM_GPU_RESOURCE_PREFIX =
1570 NM_RESOURCE_PLUGINS + ".gpu.";
1571
1572 @Private
1573 public static final String NM_GPU_ALLOWED_DEVICES =
1574 NM_GPU_RESOURCE_PREFIX + "allowed-gpu-devices";
1575 @Private
1576 public static final String AUTOMATICALLY_DISCOVER_GPU_DEVICES = "auto";
1577
1578 /**
1579 * This setting controls where to how to invoke GPU binaries
1580 */
1581 @Private
1582 public static final String NM_GPU_PATH_TO_EXEC =
1583 NM_GPU_RESOURCE_PREFIX + "path-to-discovery-executables";
1584
1585 @Private
1586 public static final String DEFAULT_NM_GPU_PATH_TO_EXEC = "";
1587
1588 /**
1589 * Settings to control which implementation of docker plugin for GPU will be
1590 * used.
1591 *
1592 * By default uses NVIDIA docker v1.
1593 */
1594 @Private
1595 public static final String NM_GPU_DOCKER_PLUGIN_IMPL =
1596 NM_GPU_RESOURCE_PREFIX + "docker-plugin";
1597
1598 @Private
1599 public static final String NVIDIA_DOCKER_V1 = "nvidia-docker-v1";
1600
1601 @Private
1602 public static final String DEFAULT_NM_GPU_DOCKER_PLUGIN_IMPL =
1603 NVIDIA_DOCKER_V1;
1604
1605 /**
1606 * This setting controls end point of nvidia-docker-v1 plugin
1607 */
1608 @Private
1609 public static final String NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT =
1610 NM_GPU_RESOURCE_PREFIX + "docker-plugin." + NVIDIA_DOCKER_V1
1611 + ".endpoint";
1612
1613 @Private
1614 public static final String DEFAULT_NVIDIA_DOCKER_PLUGIN_V1_ENDPOINT =
1615 "http://localhost:3476/v1.0/docker/cli";
1616
1617 /**
1618 * Prefix for FPGA configurations. Work in progress: This configuration
1619 * parameter may be changed/removed in the future.
1620 */
1621 @Private
1622 public static final String NM_FPGA_RESOURCE_PREFIX =
1623 NM_RESOURCE_PLUGINS + ".fpga.";
1624
1625 @Private
1626 public static final String NM_FPGA_ALLOWED_DEVICES =
1627 NM_FPGA_RESOURCE_PREFIX + "allowed-fpga-devices";
1628
1629 @Private
1630 public static final String NM_FPGA_PATH_TO_EXEC =
1631 NM_FPGA_RESOURCE_PREFIX + "path-to-discovery-executables";
1632
1633 @Private
1634 public static final String NM_FPGA_VENDOR_PLUGIN =
1635 NM_FPGA_RESOURCE_PREFIX + "vendor-plugin.class";
1636
1637 @Private
1638 public static final String DEFAULT_NM_FPGA_VENDOR_PLUGIN =
1639 "org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.IntelFpgaOpenclPlugin";
1640
1641 public static final String NM_NETWORK_TAG_PREFIX = NM_PREFIX
1642 + "network-tagging";
1643
1644 public static final String NM_NETWORK_TAG_HANDLER_ENABLED =
1645 NM_NETWORK_TAG_PREFIX + "-handler.enabled";
1646
1647 public static final boolean DEFAULT_NM_NETWORK_TAG_HANDLER_ENABLED =
1648 false;
1649
1650 public static final String NM_NETWORK_TAG_MAPPING_MANAGER =
1651 NM_NETWORK_TAG_PREFIX + ".mapping-mamager.class";
1652
1653 public static final String NM_NETWORK_TAG_MAPPING_FILE_PATH =
1654 NM_NETWORK_TAG_PREFIX + ".mapping-file.path";
1655
1656 public static final String DEFAULT_NM_NETWORK_RESOURCE_TAG_MAPPING_FILE_PATH =
1657 "";
1658
1659 /** NM Webapp address.**/
1660 public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address";
1661 public static final int DEFAULT_NM_WEBAPP_PORT = 8042;
1662 public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:" +
1663 DEFAULT_NM_WEBAPP_PORT;
1664
1665 /** NM Webapp https address.**/
1666 public static final String NM_WEBAPP_HTTPS_ADDRESS = NM_PREFIX
1667 + "webapp.https.address";
1668 public static final int DEFAULT_NM_WEBAPP_HTTPS_PORT = 8044;
1669 public static final String DEFAULT_NM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:"
1670 + DEFAULT_NM_WEBAPP_HTTPS_PORT;
1671
1672 /** Enable/disable CORS filter. */
1673 public static final String NM_WEBAPP_ENABLE_CORS_FILTER =
1674 NM_PREFIX + "webapp.cross-origin.enabled";
1675 public static final boolean DEFAULT_NM_WEBAPP_ENABLE_CORS_FILTER = false;
1676
1677 /** How often to monitor resource in a node.*/
1678 public static final String NM_RESOURCE_MON_INTERVAL_MS =
1679 NM_PREFIX + "resource-monitor.interval-ms";
1680 public static final int DEFAULT_NM_RESOURCE_MON_INTERVAL_MS = 3000;
1681
1682 public static final String NM_CONTAINER_MONITOR_ENABLED =
1683 NM_PREFIX + "container-monitor.enabled";
1684 public static final boolean DEFAULT_NM_CONTAINER_MONITOR_ENABLED = true;
1685
1686 /** How often to monitor containers.*/
1687 public final static String NM_CONTAINER_MON_INTERVAL_MS =
1688 NM_PREFIX + "container-monitor.interval-ms";
1689 @Deprecated
1690 public final static int DEFAULT_NM_CONTAINER_MON_INTERVAL_MS = 3000;
1691
1692 /** Class that calculates current resource utilization.*/
1693 public static final String NM_MON_RESOURCE_CALCULATOR =
1694 NM_PREFIX + "resource-calculator.class";
1695 /** Class that calculates containers current resource utilization.*/
1696 public static final String NM_CONTAINER_MON_RESOURCE_CALCULATOR =
1697 NM_PREFIX + "container-monitor.resource-calculator.class";
1698 /** Class that calculates process tree resource utilization.*/
1699 public static final String NM_CONTAINER_MON_PROCESS_TREE =
1700 NM_PREFIX + "container-monitor.process-tree.class";
1701 public static final String PROCFS_USE_SMAPS_BASED_RSS_ENABLED = NM_PREFIX +
1702 "container-monitor.procfs-tree.smaps-based-rss.enabled";
1703 public static final boolean DEFAULT_PROCFS_USE_SMAPS_BASED_RSS_ENABLED =
1704 false;
1705
1706 /** Enable/disable container metrics. */
1707 @Private
1708 public static final String NM_CONTAINER_METRICS_ENABLE =
1709 NM_PREFIX + "container-metrics.enable";
1710 @Private
1711 public static final boolean DEFAULT_NM_CONTAINER_METRICS_ENABLE = true;
1712
1713 /** Container metrics flush period. -1 for flush on completion. */
1714 @Private
1715 public static final String NM_CONTAINER_METRICS_PERIOD_MS =
1716 NM_PREFIX + "container-metrics.period-ms";
1717 @Private
1718 public static final int DEFAULT_NM_CONTAINER_METRICS_PERIOD_MS = -1;
1719
1720 /** The delay time ms to unregister container metrics after completion. */
1721 @Private
1722 public static final String NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS =
1723 NM_PREFIX + "container-metrics.unregister-delay-ms";
1724 @Private
1725 public static final int DEFAULT_NM_CONTAINER_METRICS_UNREGISTER_DELAY_MS =
1726 10000;
1727
1728 /** Prefix for all node manager disk health checker configs. */
1729 private static final String NM_DISK_HEALTH_CHECK_PREFIX =
1730 "yarn.nodemanager.disk-health-checker.";
1731 /**
1732 * Enable/Disable disks' health checker. Default is true. An expert level
1733 * configuration property.
1734 */
1735 public static final String NM_DISK_HEALTH_CHECK_ENABLE =
1736 NM_DISK_HEALTH_CHECK_PREFIX + "enable";
1737 /** Frequency of running disks' health checker. */
1738 public static final String NM_DISK_HEALTH_CHECK_INTERVAL_MS =
1739 NM_DISK_HEALTH_CHECK_PREFIX + "interval-ms";
1740 /** By default, disks' health is checked every 2 minutes. */
1741 public static final long DEFAULT_NM_DISK_HEALTH_CHECK_INTERVAL_MS =
1742 2 * 60 * 1000;
1743
1744 /**
1745 * The minimum fraction of number of disks to be healthy for the nodemanager
1746 * to launch new containers. This applies to nm-local-dirs and nm-log-dirs.
1747 */
1748 public static final String NM_MIN_HEALTHY_DISKS_FRACTION =
1749 NM_DISK_HEALTH_CHECK_PREFIX + "min-healthy-disks";
1750 /**
1751 * By default, at least 25% of disks are to be healthy to say that the node is
1752 * healthy in terms of disks.
1753 */
1754 public static final float DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION = 0.25F;
1755
1756 /**
1757 * The maximum percentage of disk space that can be used after which a disk is
1758 * marked as offline. Values can range from 0.0 to 100.0. If the value is
1759 * greater than or equal to 100, NM will check for full disk. This applies to
1760 * nm-local-dirs and nm-log-dirs.
1761 */
1762 public static final String NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE =
1763 NM_DISK_HEALTH_CHECK_PREFIX + "max-disk-utilization-per-disk-percentage";
1764 /**
1765 * By default, 90% of the disk can be used before it is marked as offline.
1766 */
1767 public static final float DEFAULT_NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE =
1768 90.0F;
1769
1770 /**
1771 * The low threshold percentage of disk space used when an offline disk is
1772 * marked as online. Values can range from 0.0 to 100.0. The value shouldn't
1773 * be more than NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE. If its value is
1774 * more than NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE or not set, it will be
1775 * set to the same value as NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE.
1776 * This applies to nm-local-dirs and nm-log-dirs.
1777 */
1778 public static final String NM_WM_LOW_PER_DISK_UTILIZATION_PERCENTAGE =
1779 NM_DISK_HEALTH_CHECK_PREFIX +
1780 "disk-utilization-watermark-low-per-disk-percentage";
1781
1782 /**
1783 * The minimum space that must be available on a local dir for it to be used.
1784 * This applies to nm-local-dirs and nm-log-dirs.
1785 */
1786 public static final String NM_MIN_PER_DISK_FREE_SPACE_MB =
1787 NM_DISK_HEALTH_CHECK_PREFIX + "min-free-space-per-disk-mb";
1788 /**
1789 * By default, all of the disk can be used before it is marked as offline.
1790 */
1791 public static final long DEFAULT_NM_MIN_PER_DISK_FREE_SPACE_MB = 0;
1792
1793 /** Frequency of running node health script.*/
1794 public static final String NM_HEALTH_CHECK_INTERVAL_MS =
1795 NM_PREFIX + "health-checker.interval-ms";
1796 public static final long DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS = 10 * 60 * 1000;
1797
1798 /** Health check script time out period.*/
1799 public static final String NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS =
1800 NM_PREFIX + "health-checker.script.timeout-ms";
1801 public static final long DEFAULT_NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS =
1802 2 * DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS;
1803
1804 /** The health check script to run.*/
1805 public static final String NM_HEALTH_CHECK_SCRIPT_PATH =
1806 NM_PREFIX + "health-checker.script.path";
1807
1808 /** The arguments to pass to the health check script.*/
1809 public static final String NM_HEALTH_CHECK_SCRIPT_OPTS =
1810 NM_PREFIX + "health-checker.script.opts";
1811
1812 /** The JVM options used on forking ContainerLocalizer process
1813 by container executor. */
1814 public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_KEY =
1815 NM_PREFIX + "container-localizer.java.opts";
1816 public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT =
1817 "-Xmx256m";
1818
1819 /** The log level of container localizer process. */
1820 public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL=
1821 NM_PREFIX + "container-localizer.log.level";
1822 public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT =
1823 "INFO";
1824
1825 /** Prefix for runtime configuration constants. */
1826 public static final String LINUX_CONTAINER_RUNTIME_PREFIX = NM_PREFIX +
1827 "runtime.linux.";
1828
1829 /**
1830 * Comma separated list of runtimes that are allowed when using
1831 * LinuxContainerExecutor. The allowed values are:
1832 * <ul>
1833 * <li>default</li>
1834 * <li>docker</li>
1835 * <li>javasandbox</li>
1836 * </ul>
1837 */
1838 public static final String LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES =
1839 LINUX_CONTAINER_RUNTIME_PREFIX + "allowed-runtimes";
1840
1841 /** The default list of allowed runtimes when using LinuxContainerExecutor. */
1842 public static final String[] DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES
1843 = {"default"};
1844
1845 public static final String DOCKER_CONTAINER_RUNTIME_PREFIX =
1846 LINUX_CONTAINER_RUNTIME_PREFIX + "docker.";
1847
1848 /** Capabilities allowed (and added by default) for docker containers. **/
1849 public static final String NM_DOCKER_CONTAINER_CAPABILITIES =
1850 DOCKER_CONTAINER_RUNTIME_PREFIX + "capabilities";
1851
1852 /** These are the default capabilities added by docker. We'll use the same
1853 * set here. While these may not be case-sensitive from a docker
1854 * perspective, it is best to keep these uppercase.
1855 */
1856 public static final String[] DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES = {
1857 "CHOWN",
1858 "DAC_OVERRIDE",
1859 "FSETID",
1860 "FOWNER",
1861 "MKNOD",
1862 "NET_RAW",
1863 "SETGID",
1864 "SETUID",
1865 "SETFCAP",
1866 "SETPCAP",
1867 "NET_BIND_SERVICE",
1868 "SYS_CHROOT",
1869 "KILL",
1870 "AUDIT_WRITE" };
1871
1872 /** Allow privileged containers. Use with extreme care. */
1873 public static final String NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS =
1874 DOCKER_CONTAINER_RUNTIME_PREFIX + "privileged-containers.allowed";
1875
1876 /** Privileged containers are disabled by default. */
1877 public static final boolean DEFAULT_NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS =
1878 false;
1879
1880 /** enable user remapping. */
1881 public static final String NM_DOCKER_ENABLE_USER_REMAPPING =
1882 DOCKER_CONTAINER_RUNTIME_PREFIX + "enable-userremapping.allowed";
1883
1884 /** Set enable user remapping as false by default. */
1885 public static final boolean DEFAULT_NM_DOCKER_ENABLE_USER_REMAPPING = true;
1886
1887 /** lower limit for acceptable uids of user remapped user. */
1888 public static final String NM_DOCKER_USER_REMAPPING_UID_THRESHOLD =
1889 DOCKER_CONTAINER_RUNTIME_PREFIX + "userremapping-uid-threshold";
1890
1891 /** Set user remapping lower uid limit to 1 by default. */
1892 public static final int DEFAULT_NM_DOCKER_USER_REMAPPING_UID_THRESHOLD = 1;
1893
1894 /** lower limit for acceptable gids of user remapped user. */
1895 public static final String NM_DOCKER_USER_REMAPPING_GID_THRESHOLD =
1896 DOCKER_CONTAINER_RUNTIME_PREFIX + "userremapping-gid-threshold";
1897
1898 /** Set user remapping lower gid limit to 1 by default. */
1899 public static final int DEFAULT_NM_DOCKER_USER_REMAPPING_GID_THRESHOLD = 1;
1900
1901 /** ACL list for users allowed to run privileged containers. */
1902 public static final String NM_DOCKER_PRIVILEGED_CONTAINERS_ACL =
1903 DOCKER_CONTAINER_RUNTIME_PREFIX + "privileged-containers.acl";
1904
1905 /** Default list for users allowed to run privileged containers is empty. */
1906 public static final String DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL = "";
1907
1908 /** The set of networks allowed when launching containers using the
1909 * DockerContainerRuntime. */
1910 public static final String NM_DOCKER_ALLOWED_CONTAINER_NETWORKS =
1911 DOCKER_CONTAINER_RUNTIME_PREFIX + "allowed-container-networks";
1912
1913 /** The set of networks allowed when launching containers using the
1914 * DockerContainerRuntime. */
1915 public static final String[] DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS =
1916 {"host", "none", "bridge"};
1917
1918 /** The network used when launching containers using the
1919 * DockerContainerRuntime when no network is specified in the request. This
1920 * network must be one of the (configurable) set of allowed container
1921 * networks. */
1922 public static final String NM_DOCKER_DEFAULT_CONTAINER_NETWORK =
1923 DOCKER_CONTAINER_RUNTIME_PREFIX + "default-container-network";
1924
1925 /** The network used when launching containers using the
1926 * DockerContainerRuntime when no network is specified in the request and
1927 * no default network is configured.
1928 * . */
1929 public static final String DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK =
1930 "host";
1931
1932 /** Allow host pid namespace for containers. Use with care. */
1933 public static final String NM_DOCKER_ALLOW_HOST_PID_NAMESPACE =
1934 DOCKER_CONTAINER_RUNTIME_PREFIX + "host-pid-namespace.allowed";
1935
1936 /** Host pid namespace for containers is disabled by default. */
1937 public static final boolean DEFAULT_NM_DOCKER_ALLOW_HOST_PID_NAMESPACE =
1938 false;
1939
1940 /**
1941 * Whether or not users are allowed to request that Docker containers honor
1942 * the debug deletion delay. This is useful for troubleshooting Docker
1943 * container related launch failures.
1944 */
1945 public static final String NM_DOCKER_ALLOW_DELAYED_REMOVAL =
1946 DOCKER_CONTAINER_RUNTIME_PREFIX + "delayed-removal.allowed";
1947
1948 /**
1949 * The default value on whether or not a user can request that Docker
1950 * containers honor the debug deletion delay.
1951 */
1952 public static final boolean DEFAULT_NM_DOCKER_ALLOW_DELAYED_REMOVAL = false;
1953
1954 /** The mode in which the Java Container Sandbox should run detailed by
1955 * the JavaSandboxLinuxContainerRuntime. */
1956 public static final String YARN_CONTAINER_SANDBOX =
1957 LINUX_CONTAINER_RUNTIME_PREFIX + "sandbox-mode";
1958 public static final String DEFAULT_YARN_CONTAINER_SANDBOX = "disabled";
1959
1960 /** Permissions for application local directories.*/
1961 public static final String YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS =
1962 YARN_CONTAINER_SANDBOX + ".local-dirs.permissions";
1963 public static final String DEFAULT_YARN_CONTAINER_SANDBOX_FILE_PERMISSIONS =
1964 "read";
1965
1966 /** Location for non-default java policy file.*/
1967 public static final String YARN_CONTAINER_SANDBOX_POLICY =
1968 YARN_CONTAINER_SANDBOX + ".policy";
1969
1970 /** Prefix for group to policy file mapping.*/
1971 public static final String YARN_CONTAINER_SANDBOX_POLICY_GROUP_PREFIX =
1972 YARN_CONTAINER_SANDBOX_POLICY + ".group.";
1973
1974 /** The group which will run by default without the java security manager.*/
1975 public static final String YARN_CONTAINER_SANDBOX_WHITELIST_GROUP =
1976 YARN_CONTAINER_SANDBOX + ".whitelist-group";
1977
1978 /** The path to the Linux container executor.*/
1979 public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH =
1980 NM_PREFIX + "linux-container-executor.path";
1981
1982 /**
1983 * The UNIX group that the linux-container-executor should run as.
1984 * This is intended to be set as part of container-executor.cfg.
1985 */
1986 public static final String NM_LINUX_CONTAINER_GROUP =
1987 NM_PREFIX + "linux-container-executor.group";
1988
1989 /**
1990 * True if linux-container-executor should limit itself to one user
1991 * when running in non-secure mode.
1992 */
1993 public static final String NM_NONSECURE_MODE_LIMIT_USERS = NM_PREFIX +
1994 "linux-container-executor.nonsecure-mode.limit-users";
1995
1996 public static final boolean DEFAULT_NM_NONSECURE_MODE_LIMIT_USERS = true;
1997
1998 /**
1999 * The UNIX user that containers will run as when Linux-container-executor
2000 * is used in nonsecure mode (a use case for this is using cgroups).
2001 */
2002 public static final String NM_NONSECURE_MODE_LOCAL_USER_KEY = NM_PREFIX +
2003 "linux-container-executor.nonsecure-mode.local-user";
2004
2005 public static final String DEFAULT_NM_NONSECURE_MODE_LOCAL_USER = "nobody";
2006
2007 /**
2008 * The allowed pattern for UNIX user names enforced by
2009 * Linux-container-executor when used in nonsecure mode (use case for this
2010 * is using cgroups). The default value is taken from /usr/sbin/adduser
2011 */
2012 public static final String NM_NONSECURE_MODE_USER_PATTERN_KEY = NM_PREFIX +
2013 "linux-container-executor.nonsecure-mode.user-pattern";
2014
2015 public static final String DEFAULT_NM_NONSECURE_MODE_USER_PATTERN =
2016 "^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0,255}?[$]?$";
2017
2018 /** The type of resource enforcement to use with the
2019 * linux container executor.
2020 */
2021 public static final String NM_LINUX_CONTAINER_RESOURCES_HANDLER =
2022 NM_PREFIX + "linux-container-executor.resources-handler.class";
2023
2024 /** The path the linux container executor should use for cgroups */
2025 public static final String NM_LINUX_CONTAINER_CGROUPS_HIERARCHY =
2026 NM_PREFIX + "linux-container-executor.cgroups.hierarchy";
2027
2028 /** Whether the linux container executor should mount cgroups if not found */
2029 public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT =
2030 NM_PREFIX + "linux-container-executor.cgroups.mount";
2031
2032 /** Where the linux container executor should mount cgroups if not found */
2033 public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH =
2034 NM_PREFIX + "linux-container-executor.cgroups.mount-path";
2035
2036 /**
2037 * Whether the apps should run in strict resource usage mode(not allowed to
2038 * use spare CPU)
2039 */
2040 public static final String NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE =
2041 NM_PREFIX + "linux-container-executor.cgroups.strict-resource-usage";
2042 public static final boolean DEFAULT_NM_LINUX_CONTAINER_CGROUPS_STRICT_RESOURCE_USAGE =
2043 false;
2044
2045
2046 // Configurations for applicaiton life time monitor feature
2047 public static final String RM_APPLICATION_MONITOR_INTERVAL_MS =
2048 RM_PREFIX + "application-timeouts.monitor.interval-ms";
2049
2050 public static final long DEFAULT_RM_APPLICATION_MONITOR_INTERVAL_MS =
2051 3000;
2052
2053 /**
2054 * Interval of time the linux container executor should try cleaning up
2055 * cgroups entry when cleaning up a container. This is required due to what
2056 * it seems a race condition because the SIGTERM/SIGKILL is asynch.
2057 */
2058 public static final String NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT =
2059 NM_PREFIX + "linux-container-executor.cgroups.delete-timeout-ms";
2060
2061 public static final long DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT =
2062 1000;
2063
2064 /**
2065 * Delay between attempts to remove linux cgroup.
2066 */
2067 public static final String NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY =
2068 NM_PREFIX + "linux-container-executor.cgroups.delete-delay-ms";
2069
2070 public static final long DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY =
2071 20;
2072
2073 /**
2074 * Indicates if memory and CPU limits will be set for the Windows Job
2075 * Object for the containers launched by the default container executor.
2076 */
2077 public static final String NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED =
2078 NM_PREFIX + "windows-container.memory-limit.enabled";
2079 public static final boolean DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED = false;
2080
2081 public static final String NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED =
2082 NM_PREFIX + "windows-container.cpu-limit.enabled";
2083 public static final boolean DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED = false;
2084
2085 /**
2086 /* The Windows group that the windows-secure-container-executor should run as.
2087 */
2088 public static final String NM_WINDOWS_SECURE_CONTAINER_GROUP =
2089 NM_PREFIX + "windows-secure-container-executor.group";
2090
2091 /** T-file compression types used to compress aggregated logs.*/
2092 public static final String NM_LOG_AGG_COMPRESSION_TYPE =
2093 NM_PREFIX + "log-aggregation.compression-type";
2094 public static final String DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE = "none";
2095
2096 /** The kerberos principal for the node manager.*/
2097 public static final String NM_PRINCIPAL =
2098 NM_PREFIX + "principal";
2099
2100 public static final String NM_AUX_SERVICES =
2101 NM_PREFIX + "aux-services";
2102
2103 public static final String NM_AUX_SERVICE_FMT =
2104 NM_PREFIX + "aux-services.%s.class";
2105
2106 public static final String NM_AUX_SERVICES_CLASSPATH =
2107 NM_AUX_SERVICES + ".%s.classpath";
2108
2109 public static final String NM_AUX_SERVICES_SYSTEM_CLASSES =
2110 NM_AUX_SERVICES + ".%s.system-classes";
2111
2112 public static final String NM_USER_HOME_DIR =
2113 NM_PREFIX + "user-home-dir";
2114
2115 public static final String NM_CONTAINER_STDERR_PATTERN =
2116 NM_PREFIX + "container.stderr.pattern";
2117
2118 public static final String DEFAULT_NM_CONTAINER_STDERR_PATTERN =
2119 "{*stderr*,*STDERR*}";
2120
2121 public static final String NM_CONTAINER_STDERR_BYTES =
2122 NM_PREFIX + "container.stderr.tail.bytes";
2123
2124 public static final long DEFAULT_NM_CONTAINER_STDERR_BYTES = 4 * 1024;
2125
2126 /**The kerberos principal to be used for spnego filter for NM.*/
2127 public static final String NM_WEBAPP_SPNEGO_USER_NAME_KEY =
2128 NM_PREFIX + "webapp.spnego-principal";
2129
2130 /**The kerberos keytab to be used for spnego filter for NM.*/
2131 public static final String NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
2132 NM_PREFIX + "webapp.spnego-keytab-file";
2133
2134 public static final String DEFAULT_NM_USER_HOME_DIR= "/home/";
2135
2136 public static final String NM_RECOVERY_PREFIX = NM_PREFIX + "recovery.";
2137 public static final String NM_RECOVERY_ENABLED =
2138 NM_RECOVERY_PREFIX + "enabled";
2139 public static final boolean DEFAULT_NM_RECOVERY_ENABLED = false;
2140
2141 public static final String NM_RECOVERY_DIR = NM_RECOVERY_PREFIX + "dir";
2142
2143 /** The time in seconds between full compactions of the NM state database.
2144 * Setting the interval to zero disables the full compaction cycles.
2145 */
2146 public static final String NM_RECOVERY_COMPACTION_INTERVAL_SECS =
2147 NM_RECOVERY_PREFIX + "compaction-interval-secs";
2148 public static final int DEFAULT_NM_RECOVERY_COMPACTION_INTERVAL_SECS = 3600;
2149
2150 public static final String NM_RECOVERY_SUPERVISED =
2151 NM_RECOVERY_PREFIX + "supervised";
2152 public static final boolean DEFAULT_NM_RECOVERY_SUPERVISED = false;
2153
2154 public static final String NM_LOG_AGG_POLICY_CLASS =
2155 NM_PREFIX + "log-aggregation.policy.class";
2156
2157 public static final String NM_LOG_AGG_POLICY_CLASS_PARAMETERS = NM_PREFIX
2158 + "log-aggregation.policy.parameters";
2159
2160 ////////////////////////////////
2161 // Web Proxy Configs
2162 ////////////////////////////////
2163 public static final String PROXY_PREFIX = "yarn.web-proxy.";
2164
2165 /** The kerberos principal for the proxy.*/
2166 public static final String PROXY_PRINCIPAL =
2167 PROXY_PREFIX + "principal";
2168
2169 /** Keytab for Proxy.*/
2170 public static final String PROXY_KEYTAB = PROXY_PREFIX + "keytab";
2171
2172 /** The address for the web proxy.*/
2173 public static final String PROXY_ADDRESS =
2174 PROXY_PREFIX + "address";
2175 public static final int DEFAULT_PROXY_PORT = 9099;
2176 public static final String DEFAULT_PROXY_ADDRESS =
2177 "0.0.0.0:" + DEFAULT_PROXY_PORT;
2178
2179 /**
2180 * YARN Service Level Authorization
2181 */
2182 public static final String
2183 YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL =
2184 "security.resourcetracker.protocol.acl";
2185 public static final String
2186 YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL =
2187 "security.applicationclient.protocol.acl";
2188 public static final String
2189 YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL =
2190 "security.resourcemanager-administration.protocol.acl";
2191 public static final String
2192 YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL =
2193 "security.applicationmaster.protocol.acl";
2194
2195 public static final String
2196 YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL =
2197 "security.containermanagement.protocol.acl";
2198 public static final String
2199 YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER =
2200 "security.resourcelocalizer.protocol.acl";
2201
2202 public static final String
2203 YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONHISTORY_PROTOCOL =
2204 "security.applicationhistory.protocol.acl";
2205
2206 public static final String
2207 YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL =
2208 "security.collector-nodemanager.protocol.acl";
2209
2210 /** No. of milliseconds to wait between sending a SIGTERM and SIGKILL
2211 * to a running container */
2212 public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS =
2213 NM_PREFIX + "sleep-delay-before-sigkill.ms";
2214 public static final long DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS =
2215 250;
2216
2217 /** Max time to wait for a process to come up when trying to cleanup
2218 * container resources */
2219 public static final String NM_PROCESS_KILL_WAIT_MS =
2220 NM_PREFIX + "process-kill-wait.ms";
2221 public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS =
2222 5000;
2223
2224 /** Max time to wait to establish a connection to RM */
2225 public static final String RESOURCEMANAGER_CONNECT_MAX_WAIT_MS =
2226 RM_PREFIX + "connect.max-wait.ms";
2227 public static final long DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS =
2228 15 * 60 * 1000;
2229
2230 /** Time interval between each attempt to connect to RM */
2231 public static final String RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS =
2232 RM_PREFIX + "connect.retry-interval.ms";
2233 public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS
2234 = 30 * 1000;
2235
2236 public static final String DISPATCHER_DRAIN_EVENTS_TIMEOUT =
2237 YARN_PREFIX + "dispatcher.drain-events.timeout";
2238
2239 public static final long DEFAULT_DISPATCHER_DRAIN_EVENTS_TIMEOUT = 300000;
2240
2241 /**
2242 * CLASSPATH for YARN applications. A comma-separated list of CLASSPATH
2243 * entries
2244 */
2245 public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX
2246 + "application.classpath";
2247
2248 /** The setting that controls whether AMRMProxy is enabled or not. */
2249 public static final String AMRM_PROXY_ENABLED = NM_PREFIX
2250 + "amrmproxy.enabled";
2251 public static final boolean DEFAULT_AMRM_PROXY_ENABLED = false;
2252
2253 public static final String AMRM_PROXY_ADDRESS = NM_PREFIX
2254 + "amrmproxy.address";
2255 public static final int DEFAULT_AMRM_PROXY_PORT = 8049;
2256 public static final String DEFAULT_AMRM_PROXY_ADDRESS = "0.0.0.0:"
2257 + DEFAULT_AMRM_PROXY_PORT;
2258
2259 public static final String AMRM_PROXY_CLIENT_THREAD_COUNT = NM_PREFIX
2260 + "amrmproxy.client.thread-count";
2261 public static final int DEFAULT_AMRM_PROXY_CLIENT_THREAD_COUNT = 3;
2262
2263 public static final String AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE =
2264 NM_PREFIX + "amrmproxy.interceptor-class.pipeline";
2265 public static final String DEFAULT_AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE =
2266 "org.apache.hadoop.yarn.server.nodemanager.amrmproxy."
2267 + "DefaultRequestInterceptor";
2268 public static final String AMRM_PROXY_HA_ENABLED = NM_PREFIX
2269 + "amrmproxy.ha.enable";
2270 public static final boolean DEFAULT_AMRM_PROXY_HA_ENABLED = false;
2271
2272 /**
2273 * Default platform-agnostic CLASSPATH for YARN applications. A
2274 * comma-separated list of CLASSPATH entries. The parameter expansion marker
2275 * will be replaced with real parameter expansion marker ('%' for Windows and
2276 * '$' for Linux) by NodeManager on container launch. For example: {{VAR}}
2277 * will be replaced as $VAR on Linux, and %VAR% on Windows.
2278 */
2279 @Public
2280 @Unstable
2281 public static final String[] DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH= {
2282 ApplicationConstants.Environment.HADOOP_CONF_DIR.$$(),
2283 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$()
2284 + "/share/hadoop/common/*",
2285 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$$()
2286 + "/share/hadoop/common/lib/*",
2287 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$()
2288 + "/share/hadoop/hdfs/*",
2289 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$$()
2290 + "/share/hadoop/hdfs/lib/*",
2291 ApplicationConstants.Environment.HADOOP_YARN_HOME.$$()
2292 + "/share/hadoop/yarn/*",
2293 ApplicationConstants.Environment.HADOOP_YARN_HOME.$$()
2294 + "/share/hadoop/yarn/lib/*" };
2295 /**
2296 * <p>
2297 * Default platform-specific CLASSPATH for YARN applications. A
2298 * comma-separated list of CLASSPATH entries constructed based on the client
2299 * OS environment expansion syntax.
2300 * </p>
2301 * <p>
2302 * Note: Use {@link #DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH} for
2303 * cross-platform practice i.e. submit an application from a Windows client to
2304 * a Linux/Unix server or vice versa.
2305 * </p>
2306 */
2307 public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = {
2308 ApplicationConstants.Environment.HADOOP_CONF_DIR.$(),
2309 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$()
2310 + "/share/hadoop/common/*",
2311 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$()
2312 + "/share/hadoop/common/lib/*",
2313 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$()
2314 + "/share/hadoop/hdfs/*",
2315 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$()
2316 + "/share/hadoop/hdfs/lib/*",
2317 ApplicationConstants.Environment.HADOOP_YARN_HOME.$()
2318 + "/share/hadoop/yarn/*",
2319 ApplicationConstants.Environment.HADOOP_YARN_HOME.$()
2320 + "/share/hadoop/yarn/lib/*" };
2321
2322 /** Container temp directory */
2323 public static final String DEFAULT_CONTAINER_TEMP_DIR = "./tmp";
2324
2325 public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX
2326 + "is.minicluster";
2327
2328 public static final String YARN_MC_PREFIX = YARN_PREFIX + "minicluster.";
2329
2330 /** Whether to use fixed ports with the minicluster. */
2331 public static final String YARN_MINICLUSTER_FIXED_PORTS =
2332 YARN_MC_PREFIX + "fixed.ports";
2333
2334 /**
2335 * Default is false to be able to run tests concurrently without port
2336 * conflicts.
2337 */
2338 public static final boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false;
2339
2340 /**
2341 * Whether the NM should use RPC to connect to the RM. Default is false.
2342 * Can be set to true only when using fixed ports.
2343 */
2344 public static final String YARN_MINICLUSTER_USE_RPC = YARN_MC_PREFIX + "use-rpc";
2345 public static final boolean DEFAULT_YARN_MINICLUSTER_USE_RPC = false;
2346
2347 /**
2348 * Whether users are explicitly trying to control resource monitoring
2349 * configuration for the MiniYARNCluster. Disabled by default.
2350 */
2351 public static final String YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING =
2352 YARN_MC_PREFIX + "control-resource-monitoring";
2353 public static final boolean
2354 DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING = false;
2355
2356 /** Allow changing the memory for the NodeManager in the MiniYARNCluster */
2357 public static final String YARN_MINICLUSTER_NM_PMEM_MB =
2358 YARN_MC_PREFIX + YarnConfiguration.NM_PMEM_MB;
2359 public static final int DEFAULT_YARN_MINICLUSTER_NM_PMEM_MB = 4 * 1024;
2360
2361 /** The log directory for the containers */
2362 public static final String YARN_APP_CONTAINER_LOG_DIR =
2363 YARN_PREFIX + "app.container.log.dir";
2364
2365 public static final String YARN_APP_CONTAINER_LOG_SIZE =
2366 YARN_PREFIX + "app.container.log.filesize";
2367
2368 public static final String YARN_APP_CONTAINER_LOG_BACKUPS =
2369 YARN_PREFIX + "app.container.log.backups";
2370
2371 ////////////////////////////////
2372 // Timeline Service Configs
2373 ////////////////////////////////
2374
2375 public static final String TIMELINE_SERVICE_PREFIX =
2376 YARN_PREFIX + "timeline-service.";
2377
2378 public static final String TIMELINE_SERVICE_VERSION = TIMELINE_SERVICE_PREFIX
2379 + "version";
2380 public static final float DEFAULT_TIMELINE_SERVICE_VERSION = 1.0f;
2381
2382 public static final String TIMELINE_SERVICE_VERSIONS =
2383 TIMELINE_SERVICE_PREFIX + "versions";
2384
2385 /**
2386 * Comma separated list of names for UIs hosted in the timeline server
2387 * (For pluggable UIs).
2388 */
2389 public static final String TIMELINE_SERVICE_UI_NAMES =
2390 TIMELINE_SERVICE_PREFIX + "ui-names";
2391
2392 /** Relative web path that will serve up this UI (For pluggable UIs). */
2393 public static final String TIMELINE_SERVICE_UI_WEB_PATH_PREFIX =
2394 TIMELINE_SERVICE_PREFIX + "ui-web-path.";
2395
2396 /** Timeline client settings. */
2397 public static final String TIMELINE_SERVICE_CLIENT_PREFIX =
2398 TIMELINE_SERVICE_PREFIX + "client.";
2399
2400 /**
2401 * Path to war file or static content directory for this UI
2402 * (For pluggable UIs).
2403 */
2404 public static final String TIMELINE_SERVICE_UI_ON_DISK_PATH_PREFIX =
2405 TIMELINE_SERVICE_PREFIX + "ui-on-disk-path.";
2406
2407 /**
2408 * The setting for timeline service v1.5
2409 */
2410 public static final String TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX =
2411 TIMELINE_SERVICE_PREFIX + "entity-group-fs-store.";
2412
2413 public static final String TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CACHE_STORE =
2414 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "cache-store-class";
2415
2416 public static final String TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_ACTIVE_DIR =
2417 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "active-dir";
2418
2419 public static final String
2420 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_ACTIVE_DIR_DEFAULT =
2421 "/tmp/entity-file-history/active";
2422
2423 public static final String TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR =
2424 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "done-dir";
2425 public static final String
2426 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_DONE_DIR_DEFAULT =
2427 "/tmp/entity-file-history/done";
2428
2429 public static final String TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSES =
2430 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "group-id-plugin-classes";
2431
2432 public static final String TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSPATH =
2433 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX
2434 + "group-id-plugin-classpath";
2435
2436 public static final String TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_SYSTEM_CLASSES
2437 = TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX
2438 + "group-id-plugin-system-classes";
2439
2440 public static final String
2441 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_STORE =
2442 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "summary-store";
2443
2444 public static final String
2445 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SUMMARY_ENTITY_TYPES =
2446 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "summary-entity-types";
2447
2448 public static final String
2449 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SCAN_INTERVAL_SECONDS =
2450 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "scan-interval-seconds";
2451 public static final long
2452 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SCAN_INTERVAL_SECONDS_DEFAULT = 60;
2453
2454 public static final String TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_THREADS =
2455 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "threads";
2456 public static final int
2457 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_THREADS_DEFAULT = 16;
2458
2459 public static final String
2460 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_APP_CACHE_SIZE
2461 = TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "app-cache-size";
2462 public static final int
2463 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_APP_CACHE_SIZE_DEFAULT = 10;
2464
2465 public static final String
2466 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CLEANER_INTERVAL_SECONDS =
2467 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "cleaner-interval-seconds";
2468 public static final int
2469 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_CLEANER_INTERVAL_SECONDS_DEFAULT =
2470 60 * 60;
2471
2472 public static final String
2473 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETAIN_SECONDS
2474 = TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "retain-seconds";
2475 public static final int
2476 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_RETAIN_SECONDS_DEFAULT =
2477 7 * 24 * 60 * 60;
2478
2479 // how old the most recent log of an UNKNOWN app needs to be in the active
2480 // directory before we treat it as COMPLETED
2481 public static final String
2482 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_UNKNOWN_ACTIVE_SECONDS =
2483 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "unknown-active-seconds";
2484 public static final int
2485 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_UNKNOWN_ACTIVE_SECONDS_DEFAULT
2486 = 24 * 60 * 60;
2487
2488 public static final String TIMELINE_SERVICE_LEVELDB_CACHE_READ_CACHE_SIZE =
2489 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX
2490 + "leveldb-cache-read-cache-size";
2491
2492 public static final long
2493 DEFAULT_TIMELINE_SERVICE_LEVELDB_CACHE_READ_CACHE_SIZE = 10 * 1024 * 1024;
2494
2495 public static final String TIMELINE_SERVICE_CLIENT_FD_FLUSH_INTERVAL_SECS =
2496 TIMELINE_SERVICE_CLIENT_PREFIX + "fd-flush-interval-secs";
2497 public static final long
2498 TIMELINE_SERVICE_CLIENT_FD_FLUSH_INTERVAL_SECS_DEFAULT = 10;
2499
2500 public static final String TIMELINE_SERVICE_CLIENT_FD_CLEAN_INTERVAL_SECS =
2501 TIMELINE_SERVICE_CLIENT_PREFIX + "fd-clean-interval-secs";
2502 public static final long
2503 TIMELINE_SERVICE_CLIENT_FD_CLEAN_INTERVAL_SECS_DEFAULT = 60;
2504
2505 public static final String TIMELINE_SERVICE_CLIENT_FD_RETAIN_SECS =
2506 TIMELINE_SERVICE_CLIENT_PREFIX + "fd-retain-secs";
2507 public static final long TIMELINE_SERVICE_CLIENT_FD_RETAIN_SECS_DEFAULT =
2508 5*60;
2509
2510 public static final String
2511 TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS =
2512 TIMELINE_SERVICE_CLIENT_PREFIX + "internal-timers-ttl-secs";
2513 public static final long
2514 TIMELINE_SERVICE_CLIENT_INTERNAL_TIMERS_TTL_SECS_DEFAULT = 7 * 60;
2515
2516 public static final String
2517 TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE =
2518 TIMELINE_SERVICE_CLIENT_PREFIX + "internal-attempt-dir-cache-size";
2519 public static final int
2520 DEFAULT_TIMELINE_SERVICE_CLIENT_INTERNAL_ATTEMPT_DIR_CACHE_SIZE = 1000;
2521
2522 // This is temporary solution. The configuration will be deleted once we have
2523 // the FileSystem API to check whether append operation is supported or not.
2524 public static final String TIMELINE_SERVICE_ENTITYFILE_FS_SUPPORT_APPEND
2525 = TIMELINE_SERVICE_PREFIX
2526 + "entity-file.fs-support-append";
2527
2528 public static final String
2529 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_WITH_USER_DIR =
2530 TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_PREFIX + "with-user-dir";
2531
2532 /**
2533 * Settings for timeline service v2.0.
2534 */
2535 public static final String TIMELINE_SERVICE_READER_PREFIX =
2536 TIMELINE_SERVICE_PREFIX + "reader.";
2537
2538 public static final String TIMELINE_SERVICE_WRITER_CLASS =
2539 TIMELINE_SERVICE_PREFIX + "writer.class";
2540
2541 public static final String DEFAULT_TIMELINE_SERVICE_WRITER_CLASS =
2542 "org.apache.hadoop.yarn.server.timelineservice"
2543 + ".storage.HBaseTimelineWriterImpl";
2544
2545 public static final String TIMELINE_SERVICE_READER_CLASS =
2546 TIMELINE_SERVICE_READER_PREFIX + "class";
2547
2548 public static final String DEFAULT_TIMELINE_SERVICE_READER_CLASS =
2549 "org.apache.hadoop.yarn.server.timelineservice.storage" +
2550 ".HBaseTimelineReaderImpl";
2551
2552 /**
2553 * default schema prefix for hbase tables.
2554 */
2555 public static final String DEFAULT_TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX =
2556 "prod.";
2557
2558 /**
2559 * config param name to override schema prefix.
2560 */
2561 public static final String TIMELINE_SERVICE_HBASE_SCHEMA_PREFIX_NAME =
2562 TIMELINE_SERVICE_PREFIX + "hbase-schema.prefix";
2563
2564 /** The setting that controls how often the timeline collector flushes the
2565 * timeline writer.
2566 */
2567 public static final String TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS =
2568 TIMELINE_SERVICE_PREFIX + "writer.flush-interval-seconds";
2569
2570 public static final int
2571 DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS = 60;
2572
2573 /**
2574 * The name for setting that controls how long the final value of
2575 * a metric of a completed app is retained before merging
2576 * into the flow sum.
2577 */
2578 public static final String APP_FINAL_VALUE_RETENTION_THRESHOLD =
2579 TIMELINE_SERVICE_PREFIX
2580 + "hbase.coprocessor.app-final-value-retention-milliseconds";
2581
2582 /**
2583 * The name of the setting for the location of the coprocessor
2584 * jar on hdfs.
2585 */
2586 public static final String FLOW_RUN_COPROCESSOR_JAR_HDFS_LOCATION =
2587 TIMELINE_SERVICE_PREFIX
2588 + "hbase.coprocessor.jar.hdfs.location";
2589
2590 /** default hdfs location for flowrun coprocessor jar. */
2591 public static final String DEFAULT_HDFS_LOCATION_FLOW_RUN_COPROCESSOR_JAR =
2592 "/hbase/coprocessor/hadoop-yarn-server-timelineservice.jar";
2593
2594 /**
2595 * The name for setting that points to an optional HBase configuration
2596 * (hbase-site.xml file) with settings that will override the ones found on
2597 * the classpath.
2598 */
2599 public static final String TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE =
2600 TIMELINE_SERVICE_PREFIX
2601 + "hbase.configuration.file";
2602
2603 /**
2604 * The name for setting that enables or disables authentication checks
2605 * for reading timeline service v2 data.
2606 */
2607 public static final String TIMELINE_SERVICE_READ_AUTH_ENABLED =
2608 TIMELINE_SERVICE_PREFIX + "read.authentication.enabled";
2609
2610 /**
2611 * The default setting for authentication checks for reading timeline
2612 * service v2 data.
2613 */
2614 public static final Boolean DEFAULT_TIMELINE_SERVICE_READ_AUTH_ENABLED =
2615 false;
2616
2617 /**
2618 * The name for setting that lists the users and groups who are allowed
2619 * to read timeline service v2 data. It is a comma separated list of
2620 * user, followed by space, then comma separated list of groups.
2621 * It will allow this list of users and groups to read the data
2622 * and reject everyone else.
2623 */
2624 public static final String TIMELINE_SERVICE_READ_ALLOWED_USERS =
2625 TIMELINE_SERVICE_PREFIX + "read.allowed.users";
2626
2627 /**
2628 * The default value for list of the users who are allowed to read
2629 * timeline service v2 data.
2630 */
2631 public static final String DEFAULT_TIMELINE_SERVICE_READ_ALLOWED_USERS =
2632 "";
2633
2634 /**
2635 * The setting that controls how long the final value of a metric of a
2636 * completed app is retained before merging into the flow sum. Up to this time
2637 * after an application is completed out-of-order values that arrive can be
2638 * recognized and discarded at the cost of increased storage.
2639 */
2640 public static final long DEFAULT_APP_FINAL_VALUE_RETENTION_THRESHOLD = 3 * 24
2641 * 60 * 60 * 1000L;
2642
2643 public static final String ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS =
2644 TIMELINE_SERVICE_PREFIX + "app-collector.linger-period.ms";
2645
2646 public static final int DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = 1000;
2647
2648 public static final String NUMBER_OF_ASYNC_ENTITIES_TO_MERGE =
2649 TIMELINE_SERVICE_PREFIX
2650 + "timeline-client.number-of-async-entities-to-merge";
2651
2652 public static final int DEFAULT_NUMBER_OF_ASYNC_ENTITIES_TO_MERGE = 10;
2653
2654 /** default version for any flow. */
2655 public static final String DEFAULT_FLOW_VERSION = "1";
2656
2657 /**
2658 * The time period for which timeline v2 client will wait for draining
2659 * leftover entities after stop.
2660 */
2661 public static final String TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS =
2662 TIMELINE_SERVICE_CLIENT_PREFIX + "drain-entities.timeout.ms";
2663 public static final long DEFAULT_TIMELINE_V2_CLIENT_DRAIN_TIME_MILLIS
2664 = 2000L;
2665
2666 // mark app-history related configs @Private as application history is going
2667 // to be integrated into the timeline service
2668 @Private
2669 public static final String APPLICATION_HISTORY_PREFIX =
2670 TIMELINE_SERVICE_PREFIX + "generic-application-history.";
2671
2672 /**
2673 * The setting that controls whether application history service is
2674 * enabled or not.
2675 */
2676 @Private
2677 public static final String APPLICATION_HISTORY_ENABLED =
2678 APPLICATION_HISTORY_PREFIX + "enabled";
2679 @Private
2680 public static final boolean DEFAULT_APPLICATION_HISTORY_ENABLED = false;
2681
2682 /** Application history store class */
2683 @Private
2684 public static final String APPLICATION_HISTORY_STORE =
2685 APPLICATION_HISTORY_PREFIX + "store-class";
2686
2687 /** Save container meta-info in the application history store. */
2688 @Private
2689 public static final String
2690 APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO =
2691 APPLICATION_HISTORY_PREFIX + "save-non-am-container-meta-info";
2692 @Private
2693 public static final boolean
2694 DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO = true;
2695
2696 /** URI for FileSystemApplicationHistoryStore */
2697 @Private
2698 public static final String FS_APPLICATION_HISTORY_STORE_URI =
2699 APPLICATION_HISTORY_PREFIX + "fs-history-store.uri";
2700
2701 /** T-file compression types used to compress history data.*/
2702 @Private
2703 public static final String FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE =
2704 APPLICATION_HISTORY_PREFIX + "fs-history-store.compression-type";
2705 @Private
2706 public static final String
2707 DEFAULT_FS_APPLICATION_HISTORY_STORE_COMPRESSION_TYPE = "none";
2708
2709 /** The setting that controls whether timeline service is enabled or not. */
2710 public static final String TIMELINE_SERVICE_ENABLED =
2711 TIMELINE_SERVICE_PREFIX + "enabled";
2712 public static final boolean DEFAULT_TIMELINE_SERVICE_ENABLED = false;
2713
2714 /** host:port address for timeline service RPC APIs. */
2715 public static final String TIMELINE_SERVICE_ADDRESS =
2716 TIMELINE_SERVICE_PREFIX + "address";
2717 public static final int DEFAULT_TIMELINE_SERVICE_PORT = 10200;
2718 public static final String DEFAULT_TIMELINE_SERVICE_ADDRESS = "0.0.0.0:"
2719 + DEFAULT_TIMELINE_SERVICE_PORT;
2720
2721 /** The listening endpoint for the timeline service application.*/
2722 public static final String TIMELINE_SERVICE_BIND_HOST =
2723 TIMELINE_SERVICE_PREFIX + "bind-host";
2724 public static final String DEFAULT_TIMELINE_SERVICE_BIND_HOST = "0.0.0.0";
2725
2726 /** The number of threads to handle client RPC API requests. */
2727 public static final String TIMELINE_SERVICE_HANDLER_THREAD_COUNT =
2728 TIMELINE_SERVICE_PREFIX + "handler-thread-count";
2729 public static final int DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT = 10;
2730
2731
2732 /** The address of the timeline service web application.*/
2733 public static final String TIMELINE_SERVICE_WEBAPP_ADDRESS =
2734 TIMELINE_SERVICE_PREFIX + "webapp.address";
2735
2736 public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT = 8188;
2737 public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS =
2738 "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT;
2739
2740 /** The https address of the timeline service web application.*/
2741 public static final String TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
2742 TIMELINE_SERVICE_PREFIX + "webapp.https.address";
2743
2744 public static final int DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT = 8190;
2745 public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
2746 "0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT;
2747
2748 /**
2749 * Defines the max number of applications could be fetched using
2750 * REST API or application history protocol and shown in timeline
2751 * server web ui.
2752 */
2753 public static final String APPLICATION_HISTORY_MAX_APPS =
2754 APPLICATION_HISTORY_PREFIX + "max-applications";
2755 public static final long DEFAULT_APPLICATION_HISTORY_MAX_APPS = 10000;
2756
2757 /** Timeline service store class. */
2758 public static final String TIMELINE_SERVICE_STORE =
2759 TIMELINE_SERVICE_PREFIX + "store-class";
2760
2761 /** Timeline service enable data age off */
2762 public static final String TIMELINE_SERVICE_TTL_ENABLE =
2763 TIMELINE_SERVICE_PREFIX + "ttl-enable";
2764
2765 /** Timeline service length of time to retain data */
2766 public static final String TIMELINE_SERVICE_TTL_MS =
2767 TIMELINE_SERVICE_PREFIX + "ttl-ms";
2768
2769 public static final long DEFAULT_TIMELINE_SERVICE_TTL_MS =
2770 1000 * 60 * 60 * 24 * 7;
2771
2772 /** Timeline service rolling period. Valid values are daily, half_daily,
2773 * quarter_daily, and hourly. */
2774 public static final String TIMELINE_SERVICE_ROLLING_PERIOD =
2775 TIMELINE_SERVICE_PREFIX + "rolling-period";
2776
2777 /** Roll a new database each hour. */
2778 public static final String DEFAULT_TIMELINE_SERVICE_ROLLING_PERIOD =
2779 "hourly";
2780
2781 /** Implementation specific configuration prefix for Timeline Service
2782 * leveldb.
2783 */
2784 public static final String TIMELINE_SERVICE_LEVELDB_PREFIX =
2785 TIMELINE_SERVICE_PREFIX + "leveldb-timeline-store.";
2786
2787 /** Timeline service leveldb path */
2788 public static final String TIMELINE_SERVICE_LEVELDB_PATH =
2789 TIMELINE_SERVICE_LEVELDB_PREFIX + "path";
2790
2791 /** Timeline service leveldb read cache (uncompressed blocks). This is
2792 * per rolling instance so should be tuned if using rolling leveldb
2793 * timeline store */
2794 public static final String TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE =
2795 TIMELINE_SERVICE_LEVELDB_PREFIX + "read-cache-size";
2796
2797 /** Default leveldb read cache size if no configuration is specified. */
2798 public static final long DEFAULT_TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE =
2799 100 * 1024 * 1024;
2800
2801 /** Timeline service leveldb write buffer size. */
2802 public static final String TIMELINE_SERVICE_LEVELDB_WRITE_BUFFER_SIZE =
2803 TIMELINE_SERVICE_LEVELDB_PREFIX + "write-buffer-size";
2804
2805 /** Default leveldb write buffer size if no configuration is specified. This
2806 * is per rolling instance so should be tuned if using rolling leveldb
2807 * timeline store. */
2808 public static final int DEFAULT_TIMELINE_SERVICE_LEVELDB_WRITE_BUFFER_SIZE =
2809 16 * 1024 * 1024;
2810
2811 /** Timeline service leveldb write batch size. This value can be tuned down
2812 * to reduce lock time for ttl eviction. */
2813 public static final String
2814 TIMELINE_SERVICE_LEVELDB_WRITE_BATCH_SIZE =
2815 TIMELINE_SERVICE_LEVELDB_PREFIX + "write-batch-size";
2816
2817 /** Default leveldb write batch size is no configuration is specified */
2818 public static final int
2819 DEFAULT_TIMELINE_SERVICE_LEVELDB_WRITE_BATCH_SIZE = 10000;
2820
2821 /** Timeline service leveldb start time read cache (number of entities) */
2822 public static final String
2823 TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE =
2824 TIMELINE_SERVICE_LEVELDB_PREFIX + "start-time-read-cache-size";
2825
2826 public static final int
2827 DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE = 10000;
2828
2829 /** Timeline service leveldb start time write cache (number of entities) */
2830 public static final String
2831 TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE =
2832 TIMELINE_SERVICE_LEVELDB_PREFIX + "start-time-write-cache-size";
2833
2834 public static final int
2835 DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE = 10000;
2836
2837 /** Timeline service leveldb interval to wait between deletion rounds */
2838 public static final String TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS =
2839 TIMELINE_SERVICE_LEVELDB_PREFIX + "ttl-interval-ms";
2840
2841 public static final long DEFAULT_TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS =
2842 1000 * 60 * 5;
2843
2844 /** Timeline service leveldb number of concurrent open files. Tuned this
2845 * configuration to stay within system limits. This is per rolling instance
2846 * so should be tuned if using rolling leveldb timeline store. */
2847 public static final String TIMELINE_SERVICE_LEVELDB_MAX_OPEN_FILES =
2848 TIMELINE_SERVICE_LEVELDB_PREFIX + "max-open-files";
2849
2850 /** Default leveldb max open files if no configuration is specified. */
2851 public static final int DEFAULT_TIMELINE_SERVICE_LEVELDB_MAX_OPEN_FILES =
2852 1000;
2853
2854 /** The Kerberos principal for the timeline server.*/
2855 public static final String TIMELINE_SERVICE_PRINCIPAL =
2856 TIMELINE_SERVICE_PREFIX + "principal";
2857
2858 /** The Kerberos keytab for the timeline server.*/
2859 public static final String TIMELINE_SERVICE_KEYTAB =
2860 TIMELINE_SERVICE_PREFIX + "keytab";
2861
2862 /** Enables cross origin support for timeline server.*/
2863 public static final String TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED =
2864 TIMELINE_SERVICE_PREFIX + "http-cross-origin.enabled";
2865
2866 /** Default value for cross origin support for timeline server.*/
2867 public static final boolean
2868 TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT = false;
2869
2870 /** Timeline client call, max retries (-1 means no limit) */
2871 public static final String TIMELINE_SERVICE_CLIENT_MAX_RETRIES =
2872 TIMELINE_SERVICE_CLIENT_PREFIX + "max-retries";
2873
2874 public static final int DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES = 30;
2875
2876 /** Timeline client call, retry interval */
2877 public static final String TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS =
2878 TIMELINE_SERVICE_CLIENT_PREFIX + "retry-interval-ms";
2879
2880 public static final long
2881 DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS = 1000;
2882
2883 /** Timeline client policy for whether connections are fatal */
2884 public static final String TIMELINE_SERVICE_CLIENT_BEST_EFFORT =
2885 TIMELINE_SERVICE_CLIENT_PREFIX + "best-effort";
2886
2887 public static final boolean
2888 DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT = false;
2889
2890 /** Flag to enable recovery of timeline service */
2891 public static final String TIMELINE_SERVICE_RECOVERY_ENABLED =
2892 TIMELINE_SERVICE_PREFIX + "recovery.enabled";
2893 public static final boolean DEFAULT_TIMELINE_SERVICE_RECOVERY_ENABLED = false;
2894
2895 /** Timeline service state store class */
2896 public static final String TIMELINE_SERVICE_STATE_STORE_CLASS =
2897 TIMELINE_SERVICE_PREFIX + "state-store-class";
2898
2899 public static final String TIMELINE_SERVICE_LEVELDB_STATE_STORE_PREFIX =
2900 TIMELINE_SERVICE_PREFIX + "leveldb-state-store.";
2901
2902 /** Timeline service state store leveldb path */
2903 public static final String TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH =
2904 TIMELINE_SERVICE_LEVELDB_STATE_STORE_PREFIX + "path";
2905
2906 // Timeline delegation token related keys
2907 public static final String TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL =
2908 TIMELINE_SERVICE_PREFIX + "delegation.key.update-interval";
2909 public static final long DEFAULT_TIMELINE_DELEGATION_KEY_UPDATE_INTERVAL =
2910 24*60*60*1000; // 1 day
2911 public static final String TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL =
2912 TIMELINE_SERVICE_PREFIX + "delegation.token.renew-interval";
2913 public static final long DEFAULT_TIMELINE_DELEGATION_TOKEN_RENEW_INTERVAL =
2914 24*60*60*1000; // 1 day
2915 public static final String TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME =
2916 TIMELINE_SERVICE_PREFIX + "delegation.token.max-lifetime";
2917 public static final long DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME =
2918 7*24*60*60*1000; // 7 days
2919
2920 // Timeline service v2 offlien aggregation related keys
2921 public static final String TIMELINE_OFFLINE_AGGREGATION_PREFIX =
2922 YarnConfiguration.TIMELINE_SERVICE_PREFIX + "aggregation.offline.";
2923 public static final String PHOENIX_OFFLINE_STORAGE_CONN_STR
2924 = TIMELINE_OFFLINE_AGGREGATION_PREFIX
2925 + "phoenix.connectionString";
2926
2927 public static final String PHOENIX_OFFLINE_STORAGE_CONN_STR_DEFAULT
2928 = "jdbc:phoenix:localhost:2181:/hbase";
2929
2930 // ///////////////////////////////
2931 // Shared Cache Configs
2932 // ///////////////////////////////
2933 public static final String SHARED_CACHE_PREFIX = "yarn.sharedcache.";
2934
2935 // common configs
2936 /** whether the shared cache is enabled/disabled */
2937 public static final String SHARED_CACHE_ENABLED =
2938 SHARED_CACHE_PREFIX + "enabled";
2939 public static final boolean DEFAULT_SHARED_CACHE_ENABLED = false;
2940
2941 /** The config key for the shared cache root directory. */
2942 public static final String SHARED_CACHE_ROOT =
2943 SHARED_CACHE_PREFIX + "root-dir";
2944 public static final String DEFAULT_SHARED_CACHE_ROOT = "/sharedcache";
2945
2946 /** The config key for the level of nested directories before getting to the
2947 * checksum directory. */
2948 public static final String SHARED_CACHE_NESTED_LEVEL =
2949 SHARED_CACHE_PREFIX + "nested-level";
2950 public static final int DEFAULT_SHARED_CACHE_NESTED_LEVEL = 3;
2951
2952 // Shared Cache Manager Configs
2953
2954 public static final String SCM_STORE_PREFIX = SHARED_CACHE_PREFIX + "store.";
2955
2956 public static final String SCM_STORE_CLASS = SCM_STORE_PREFIX + "class";
2957 public static final String DEFAULT_SCM_STORE_CLASS =
2958 "org.apache.hadoop.yarn.server.sharedcachemanager.store.InMemorySCMStore";
2959
2960 public static final String SCM_APP_CHECKER_CLASS = SHARED_CACHE_PREFIX
2961 + "app-checker.class";
2962 public static final String DEFAULT_SCM_APP_CHECKER_CLASS =
2963 "org.apache.hadoop.yarn.server.sharedcachemanager.RemoteAppChecker";
2964
2965 /** The address of the SCM admin interface. */
2966 public static final String SCM_ADMIN_ADDRESS =
2967 SHARED_CACHE_PREFIX + "admin.address";
2968 public static final int DEFAULT_SCM_ADMIN_PORT = 8047;
2969 public static final String DEFAULT_SCM_ADMIN_ADDRESS =
2970 "0.0.0.0:" + DEFAULT_SCM_ADMIN_PORT;
2971
2972 /** Number of threads used to handle SCM admin interface. */
2973 public static final String SCM_ADMIN_CLIENT_THREAD_COUNT =
2974 SHARED_CACHE_PREFIX + "admin.thread-count";
2975 public static final int DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT = 1;
2976
2977 /** The address of the SCM web application. */
2978 public static final String SCM_WEBAPP_ADDRESS =
2979 SHARED_CACHE_PREFIX + "webapp.address";
2980 public static final int DEFAULT_SCM_WEBAPP_PORT = 8788;
2981 public static final String DEFAULT_SCM_WEBAPP_ADDRESS =
2982 "0.0.0.0:" + DEFAULT_SCM_WEBAPP_PORT;
2983
2984 // In-memory SCM store configuration
2985
2986 public static final String IN_MEMORY_STORE_PREFIX =
2987 SCM_STORE_PREFIX + "in-memory.";
2988
2989 /**
2990 * A resource in the InMemorySCMStore is considered stale if the time since
2991 * the last reference exceeds the staleness period. This value is specified in
2992 * minutes.
2993 */
2994 public static final String IN_MEMORY_STALENESS_PERIOD_MINS =
2995 IN_MEMORY_STORE_PREFIX + "staleness-period-mins";
2996 public static final int DEFAULT_IN_MEMORY_STALENESS_PERIOD_MINS =
2997 7 * 24 * 60;
2998
2999 /**
3000 * Initial delay before the in-memory store runs its first check to remove
3001 * dead initial applications. Specified in minutes.
3002 */
3003 public static final String IN_MEMORY_INITIAL_DELAY_MINS =
3004 IN_MEMORY_STORE_PREFIX + "initial-delay-mins";
3005 public static final int DEFAULT_IN_MEMORY_INITIAL_DELAY_MINS = 10;
3006
3007 /**
3008 * The frequency at which the in-memory store checks to remove dead initial
3009 * applications. Specified in minutes.
3010 */
3011 public static final String IN_MEMORY_CHECK_PERIOD_MINS =
3012 IN_MEMORY_STORE_PREFIX + "check-period-mins";
3013 public static final int DEFAULT_IN_MEMORY_CHECK_PERIOD_MINS = 12 * 60;
3014
3015 // SCM Cleaner service configuration
3016
3017 private static final String SCM_CLEANER_PREFIX = SHARED_CACHE_PREFIX
3018 + "cleaner.";
3019
3020 /**
3021 * The frequency at which a cleaner task runs. Specified in minutes.
3022 */
3023 public static final String SCM_CLEANER_PERIOD_MINS =
3024 SCM_CLEANER_PREFIX + "period-mins";
3025 public static final int DEFAULT_SCM_CLEANER_PERIOD_MINS = 24 * 60;
3026
3027 /**
3028 * Initial delay before the first cleaner task is scheduled. Specified in
3029 * minutes.
3030 */
3031 public static final String SCM_CLEANER_INITIAL_DELAY_MINS =
3032 SCM_CLEANER_PREFIX + "initial-delay-mins";
3033 public static final int DEFAULT_SCM_CLEANER_INITIAL_DELAY_MINS = 10;
3034
3035 /**
3036 * The time to sleep between processing each shared cache resource. Specified
3037 * in milliseconds.
3038 */
3039 public static final String SCM_CLEANER_RESOURCE_SLEEP_MS =
3040 SCM_CLEANER_PREFIX + "resource-sleep-ms";
3041 public static final long DEFAULT_SCM_CLEANER_RESOURCE_SLEEP_MS = 0L;
3042
3043 /** The address of the node manager interface in the SCM. */
3044 public static final String SCM_UPLOADER_SERVER_ADDRESS = SHARED_CACHE_PREFIX
3045 + "uploader.server.address";
3046 public static final int DEFAULT_SCM_UPLOADER_SERVER_PORT = 8046;
3047 public static final String DEFAULT_SCM_UPLOADER_SERVER_ADDRESS = "0.0.0.0:"
3048 + DEFAULT_SCM_UPLOADER_SERVER_PORT;
3049
3050 /**
3051 * The number of SCM threads used to handle notify requests from the node
3052 * manager.
3053 */
3054 public static final String SCM_UPLOADER_SERVER_THREAD_COUNT =
3055 SHARED_CACHE_PREFIX + "uploader.server.thread-count";
3056 public static final int DEFAULT_SCM_UPLOADER_SERVER_THREAD_COUNT = 50;
3057
3058 /** The address of the client interface in the SCM. */
3059 public static final String SCM_CLIENT_SERVER_ADDRESS =
3060 SHARED_CACHE_PREFIX + "client-server.address";
3061 public static final int DEFAULT_SCM_CLIENT_SERVER_PORT = 8045;
3062 public static final String DEFAULT_SCM_CLIENT_SERVER_ADDRESS = "0.0.0.0:"
3063 + DEFAULT_SCM_CLIENT_SERVER_PORT;
3064
3065 /** The number of threads used to handle shared cache manager requests. */
3066 public static final String SCM_CLIENT_SERVER_THREAD_COUNT =
3067 SHARED_CACHE_PREFIX + "client-server.thread-count";
3068 public static final int DEFAULT_SCM_CLIENT_SERVER_THREAD_COUNT = 50;
3069
3070 /** the checksum algorithm implementation **/
3071 public static final String SHARED_CACHE_CHECKSUM_ALGO_IMPL =
3072 SHARED_CACHE_PREFIX + "checksum.algo.impl";
3073 public static final String DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL =
3074 "org.apache.hadoop.yarn.sharedcache.ChecksumSHA256Impl";
3075
3076 // node manager (uploader) configs
3077 /**
3078 * The replication factor for the node manager uploader for the shared cache.
3079 */
3080 public static final String SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR =
3081 SHARED_CACHE_PREFIX + "nm.uploader.replication.factor";
3082 public static final int DEFAULT_SHARED_CACHE_NM_UPLOADER_REPLICATION_FACTOR =
3083 10;
3084
3085 public static final String SHARED_CACHE_NM_UPLOADER_THREAD_COUNT =
3086 SHARED_CACHE_PREFIX + "nm.uploader.thread-count";
3087 public static final int DEFAULT_SHARED_CACHE_NM_UPLOADER_THREAD_COUNT = 20;
3088
3089 ////////////////////////////////
3090 // Federation Configs
3091 ////////////////////////////////
3092
3093 public static final String FEDERATION_PREFIX = YARN_PREFIX + "federation.";
3094
3095 public static final String FEDERATION_ENABLED = FEDERATION_PREFIX + "enabled";
3096 public static final boolean DEFAULT_FEDERATION_ENABLED = false;
3097
3098 public static final String FEDERATION_FAILOVER_ENABLED =
3099 FEDERATION_PREFIX + "failover.enabled";
3100 public static final boolean DEFAULT_FEDERATION_FAILOVER_ENABLED = true;
3101
3102 public static final String FEDERATION_STATESTORE_CLIENT_CLASS =
3103 FEDERATION_PREFIX + "state-store.class";
3104
3105 public static final String DEFAULT_FEDERATION_STATESTORE_CLIENT_CLASS =
3106 "org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore";
3107
3108 public static final String FEDERATION_CACHE_TIME_TO_LIVE_SECS =
3109 FEDERATION_PREFIX + "cache-ttl.secs";
3110 // 5 minutes
3111 public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
3112
3113 public static final String FEDERATION_FLUSH_CACHE_FOR_RM_ADDR =
3114 FEDERATION_PREFIX + "flush-cache-for-rm-addr";
3115 public static final boolean DEFAULT_FEDERATION_FLUSH_CACHE_FOR_RM_ADDR = true;
3116
3117 public static final String FEDERATION_REGISTRY_BASE_KEY =
3118 FEDERATION_PREFIX + "registry.base-dir";
3119 public static final String DEFAULT_FEDERATION_REGISTRY_BASE_KEY =
3120 "yarnfederation/";
3121
3122 public static final String FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS =
3123 FEDERATION_PREFIX + "state-store.heartbeat-interval-secs";
3124
3125 // 5 minutes
3126 public static final int
3127 DEFAULT_FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS =
3128 5 * 60;
3129
3130 public static final String FEDERATION_MACHINE_LIST =
3131 FEDERATION_PREFIX + "machine-list";
3132
3133 public static final String FEDERATION_CLUSTER_RESOLVER_CLASS =
3134 FEDERATION_PREFIX + "subcluster-resolver.class";
3135 public static final String DEFAULT_FEDERATION_CLUSTER_RESOLVER_CLASS =
3136 "org.apache.hadoop.yarn.server.federation.resolver."
3137 + "DefaultSubClusterResolverImpl";
3138
3139 public static final String DEFAULT_FEDERATION_POLICY_KEY = "*";
3140
3141 public static final String FEDERATION_POLICY_MANAGER = FEDERATION_PREFIX
3142 + "policy-manager";
3143
3144 public static final String DEFAULT_FEDERATION_POLICY_MANAGER = "org.apache"
3145 + ".hadoop.yarn.server.federation.policies"
3146 + ".manager.UniformBroadcastPolicyManager";
3147
3148 public static final String FEDERATION_POLICY_MANAGER_PARAMS =
3149 FEDERATION_PREFIX + "policy-manager-params";
3150
3151 public static final String DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS = "";
3152
3153 public static final String FEDERATION_STATESTORE_ZK_PREFIX =
3154 FEDERATION_PREFIX + "zk-state-store.";
3155 /** Parent znode path under which ZKRMStateStore will create znodes. */
3156 public static final String FEDERATION_STATESTORE_ZK_PARENT_PATH =
3157 FEDERATION_STATESTORE_ZK_PREFIX + "parent-path";
3158 public static final String DEFAULT_FEDERATION_STATESTORE_ZK_PARENT_PATH =
3159 "/federationstore";
3160
3161 private static final String FEDERATION_STATESTORE_SQL_PREFIX =
3162 FEDERATION_PREFIX + "state-store.sql.";
3163
3164 public static final String FEDERATION_STATESTORE_SQL_USERNAME =
3165 FEDERATION_STATESTORE_SQL_PREFIX + "username";
3166
3167 public static final String FEDERATION_STATESTORE_SQL_PASSWORD =
3168 FEDERATION_STATESTORE_SQL_PREFIX + "password";
3169
3170 public static final String FEDERATION_STATESTORE_SQL_URL =
3171 FEDERATION_STATESTORE_SQL_PREFIX + "url";
3172
3173 public static final String FEDERATION_STATESTORE_SQL_JDBC_CLASS =
3174 FEDERATION_STATESTORE_SQL_PREFIX + "jdbc-class";
3175
3176 public static final String DEFAULT_FEDERATION_STATESTORE_SQL_JDBC_CLASS =
3177 "org.hsqldb.jdbc.JDBCDataSource";
3178
3179 public static final String FEDERATION_STATESTORE_SQL_MAXCONNECTIONS =
3180 FEDERATION_STATESTORE_SQL_PREFIX + "max-connections";
3181
3182 public static final int DEFAULT_FEDERATION_STATESTORE_SQL_MAXCONNECTIONS = 1;
3183
3184 public static final String ROUTER_PREFIX = YARN_PREFIX + "router.";
3185
3186 public static final String ROUTER_BIND_HOST = ROUTER_PREFIX + "bind-host";
3187
3188 public static final String ROUTER_CLIENTRM_PREFIX =
3189 ROUTER_PREFIX + "clientrm.";
3190
3191 public static final String ROUTER_CLIENTRM_ADDRESS =
3192 ROUTER_CLIENTRM_PREFIX + "address";
3193 public static final int DEFAULT_ROUTER_CLIENTRM_PORT = 8050;
3194 public static final String DEFAULT_ROUTER_CLIENTRM_ADDRESS =
3195 "0.0.0.0:" + DEFAULT_ROUTER_CLIENTRM_PORT;
3196
3197 public static final String ROUTER_CLIENTRM_INTERCEPTOR_CLASS_PIPELINE =
3198 ROUTER_CLIENTRM_PREFIX + "interceptor-class.pipeline";
3199 public static final String DEFAULT_ROUTER_CLIENTRM_INTERCEPTOR_CLASS =
3200 "org.apache.hadoop.yarn.server.router.clientrm."
3201 + "DefaultClientRequestInterceptor";
3202
3203 public static final String ROUTER_PIPELINE_CACHE_MAX_SIZE =
3204 ROUTER_PREFIX + "pipeline.cache-max-size";
3205 public static final int DEFAULT_ROU