001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.hdfs; 020 021import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN; 022import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT; 023import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY; 024import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX; 025import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY; 026import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY; 027import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT; 028import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY; 029import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT; 030import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY; 031import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY; 032import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY; 033import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY; 034import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES; 035import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID; 036 037import java.io.IOException; 038import java.io.PrintStream; 039import java.io.UnsupportedEncodingException; 040import java.net.InetSocketAddress; 041import java.net.URI; 042import java.net.URISyntaxException; 043import java.security.SecureRandom; 044import java.text.SimpleDateFormat; 045import java.util.ArrayList; 046import java.util.Collection; 047import java.util.Collections; 048import java.util.Comparator; 049import java.util.Date; 050import java.util.HashSet; 051import java.util.List; 052import java.util.Locale; 053import java.util.Map; 054import java.util.Random; 055import java.util.Set; 056import java.util.concurrent.TimeUnit; 057 058import javax.net.SocketFactory; 059 060import org.apache.commons.cli.CommandLine; 061import org.apache.commons.cli.CommandLineParser; 062import org.apache.commons.cli.Option; 063import org.apache.commons.cli.Options; 064import org.apache.commons.cli.ParseException; 065import org.apache.commons.cli.PosixParser; 066import org.apache.commons.logging.Log; 067import org.apache.commons.logging.LogFactory; 068import org.apache.hadoop.HadoopIllegalArgumentException; 069import org.apache.hadoop.classification.InterfaceAudience; 070import org.apache.hadoop.conf.Configuration; 071import org.apache.hadoop.fs.BlockLocation; 072import org.apache.hadoop.fs.CommonConfigurationKeys; 073import org.apache.hadoop.fs.FileSystem; 074import org.apache.hadoop.fs.Path; 075import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; 076import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; 077import org.apache.hadoop.hdfs.protocol.DatanodeID; 078import org.apache.hadoop.hdfs.protocol.DatanodeInfo; 079import org.apache.hadoop.hdfs.protocol.HdfsConstants; 080import org.apache.hadoop.hdfs.protocol.LocatedBlock; 081import org.apache.hadoop.hdfs.protocol.LocatedBlocks; 082import org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolTranslatorPB; 083import org.apache.hadoop.hdfs.server.namenode.FSDirectory; 084import org.apache.hadoop.hdfs.server.namenode.NameNode; 085import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem; 086import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; 087import org.apache.hadoop.http.HttpConfig; 088import org.apache.hadoop.http.HttpServer2; 089import org.apache.hadoop.ipc.ProtobufRpcEngine; 090import org.apache.hadoop.ipc.RPC; 091import org.apache.hadoop.net.NetUtils; 092import org.apache.hadoop.net.NodeBase; 093import org.apache.hadoop.security.SecurityUtil; 094import org.apache.hadoop.security.UserGroupInformation; 095import org.apache.hadoop.security.authorize.AccessControlList; 096import org.apache.hadoop.util.StringUtils; 097import org.apache.hadoop.util.ToolRunner; 098 099import com.google.common.annotations.VisibleForTesting; 100import com.google.common.base.Charsets; 101import com.google.common.base.Joiner; 102import com.google.common.base.Preconditions; 103import com.google.common.collect.Lists; 104import com.google.common.collect.Maps; 105import com.google.common.primitives.SignedBytes; 106import com.google.protobuf.BlockingService; 107 108@InterfaceAudience.Private 109public class DFSUtil { 110 public static final Log LOG = LogFactory.getLog(DFSUtil.class.getName()); 111 112 public static final byte[] EMPTY_BYTES = {}; 113 114 /** Compare two byte arrays by lexicographical order. */ 115 public static int compareBytes(byte[] left, byte[] right) { 116 if (left == null) { 117 left = EMPTY_BYTES; 118 } 119 if (right == null) { 120 right = EMPTY_BYTES; 121 } 122 return SignedBytes.lexicographicalComparator().compare(left, right); 123 } 124 125 private DFSUtil() { /* Hidden constructor */ } 126 private static final ThreadLocal<Random> RANDOM = new ThreadLocal<Random>() { 127 @Override 128 protected Random initialValue() { 129 return new Random(); 130 } 131 }; 132 133 private static final ThreadLocal<SecureRandom> SECURE_RANDOM = new ThreadLocal<SecureRandom>() { 134 @Override 135 protected SecureRandom initialValue() { 136 return new SecureRandom(); 137 } 138 }; 139 140 /** @return a pseudo random number generator. */ 141 public static Random getRandom() { 142 return RANDOM.get(); 143 } 144 145 /** @return a pseudo secure random number generator. */ 146 public static SecureRandom getSecureRandom() { 147 return SECURE_RANDOM.get(); 148 } 149 150 /** Shuffle the elements in the given array. */ 151 public static <T> T[] shuffle(final T[] array) { 152 if (array != null && array.length > 0) { 153 final Random random = getRandom(); 154 for (int n = array.length; n > 1; ) { 155 final int randomIndex = random.nextInt(n); 156 n--; 157 if (n != randomIndex) { 158 final T tmp = array[randomIndex]; 159 array[randomIndex] = array[n]; 160 array[n] = tmp; 161 } 162 } 163 } 164 return array; 165 } 166 167 /** 168 * Compartor for sorting DataNodeInfo[] based on decommissioned states. 169 * Decommissioned nodes are moved to the end of the array on sorting with 170 * this compartor. 171 */ 172 public static final Comparator<DatanodeInfo> DECOM_COMPARATOR = 173 new Comparator<DatanodeInfo>() { 174 @Override 175 public int compare(DatanodeInfo a, DatanodeInfo b) { 176 return a.isDecommissioned() == b.isDecommissioned() ? 0 : 177 a.isDecommissioned() ? 1 : -1; 178 } 179 }; 180 181 182 /** 183 * Comparator for sorting DataNodeInfo[] based on decommissioned/stale states. 184 * Decommissioned/stale nodes are moved to the end of the array on sorting 185 * with this comparator. 186 */ 187 @InterfaceAudience.Private 188 public static class DecomStaleComparator implements Comparator<DatanodeInfo> { 189 private long staleInterval; 190 191 /** 192 * Constructor of DecomStaleComparator 193 * 194 * @param interval 195 * The time interval for marking datanodes as stale is passed from 196 * outside, since the interval may be changed dynamically 197 */ 198 public DecomStaleComparator(long interval) { 199 this.staleInterval = interval; 200 } 201 202 @Override 203 public int compare(DatanodeInfo a, DatanodeInfo b) { 204 // Decommissioned nodes will still be moved to the end of the list 205 if (a.isDecommissioned()) { 206 return b.isDecommissioned() ? 0 : 1; 207 } else if (b.isDecommissioned()) { 208 return -1; 209 } 210 // Stale nodes will be moved behind the normal nodes 211 boolean aStale = a.isStale(staleInterval); 212 boolean bStale = b.isStale(staleInterval); 213 return aStale == bStale ? 0 : (aStale ? 1 : -1); 214 } 215 } 216 217 /** 218 * Address matcher for matching an address to local address 219 */ 220 static final AddressMatcher LOCAL_ADDRESS_MATCHER = new AddressMatcher() { 221 @Override 222 public boolean match(InetSocketAddress s) { 223 return NetUtils.isLocalAddress(s.getAddress()); 224 }; 225 }; 226 227 /** 228 * Whether the pathname is valid. Currently prohibits relative paths, 229 * names which contain a ":" or "//", or other non-canonical paths. 230 */ 231 public static boolean isValidName(String src) { 232 // Path must be absolute. 233 if (!src.startsWith(Path.SEPARATOR)) { 234 return false; 235 } 236 237 // Check for ".." "." ":" "/" 238 String[] components = StringUtils.split(src, '/'); 239 for (int i = 0; i < components.length; i++) { 240 String element = components[i]; 241 if (element.equals(".") || 242 (element.indexOf(":") >= 0) || 243 (element.indexOf("/") >= 0)) { 244 return false; 245 } 246 // ".." is allowed in path starting with /.reserved/.inodes 247 if (element.equals("..")) { 248 if (components.length > 4 249 && components[1].equals(FSDirectory.DOT_RESERVED_STRING) 250 && components[2].equals(FSDirectory.DOT_INODES_STRING)) { 251 continue; 252 } 253 return false; 254 } 255 // The string may start or end with a /, but not have 256 // "//" in the middle. 257 if (element.isEmpty() && i != components.length - 1 && 258 i != 0) { 259 return false; 260 } 261 } 262 return true; 263 } 264 265 /** 266 * Converts a byte array to a string using UTF8 encoding. 267 */ 268 public static String bytes2String(byte[] bytes) { 269 return bytes2String(bytes, 0, bytes.length); 270 } 271 272 /** 273 * Decode a specific range of bytes of the given byte array to a string 274 * using UTF8. 275 * 276 * @param bytes The bytes to be decoded into characters 277 * @param offset The index of the first byte to decode 278 * @param length The number of bytes to decode 279 * @return The decoded string 280 */ 281 public static String bytes2String(byte[] bytes, int offset, int length) { 282 try { 283 return new String(bytes, offset, length, "UTF8"); 284 } catch(UnsupportedEncodingException e) { 285 assert false : "UTF8 encoding is not supported "; 286 } 287 return null; 288 } 289 290 /** 291 * Converts a string to a byte array using UTF8 encoding. 292 */ 293 public static byte[] string2Bytes(String str) { 294 return str.getBytes(Charsets.UTF_8); 295 } 296 297 /** 298 * Given a list of path components returns a path as a UTF8 String 299 */ 300 public static String byteArray2PathString(byte[][] pathComponents) { 301 if (pathComponents.length == 0) { 302 return ""; 303 } else if (pathComponents.length == 1 304 && (pathComponents[0] == null || pathComponents[0].length == 0)) { 305 return Path.SEPARATOR; 306 } 307 StringBuilder result = new StringBuilder(); 308 for (int i = 0; i < pathComponents.length; i++) { 309 result.append(new String(pathComponents[i], Charsets.UTF_8)); 310 if (i < pathComponents.length - 1) { 311 result.append(Path.SEPARATOR_CHAR); 312 } 313 } 314 return result.toString(); 315 } 316 317 /** 318 * Given a list of path components returns a byte array 319 */ 320 public static byte[] byteArray2bytes(byte[][] pathComponents) { 321 if (pathComponents.length == 0) { 322 return EMPTY_BYTES; 323 } else if (pathComponents.length == 1 324 && (pathComponents[0] == null || pathComponents[0].length == 0)) { 325 return new byte[]{(byte) Path.SEPARATOR_CHAR}; 326 } 327 int length = 0; 328 for (int i = 0; i < pathComponents.length; i++) { 329 length += pathComponents[i].length; 330 if (i < pathComponents.length - 1) { 331 length++; // for SEPARATOR 332 } 333 } 334 byte[] path = new byte[length]; 335 int index = 0; 336 for (int i = 0; i < pathComponents.length; i++) { 337 System.arraycopy(pathComponents[i], 0, path, index, 338 pathComponents[i].length); 339 index += pathComponents[i].length; 340 if (i < pathComponents.length - 1) { 341 path[index] = (byte) Path.SEPARATOR_CHAR; 342 index++; 343 } 344 } 345 return path; 346 } 347 348 /** Convert an object representing a path to a string. */ 349 public static String path2String(final Object path) { 350 return path == null? null 351 : path instanceof String? (String)path 352 : path instanceof byte[][]? byteArray2PathString((byte[][])path) 353 : path.toString(); 354 } 355 356 /** 357 * Splits the array of bytes into array of arrays of bytes 358 * on byte separator 359 * @param bytes the array of bytes to split 360 * @param separator the delimiting byte 361 */ 362 public static byte[][] bytes2byteArray(byte[] bytes, byte separator) { 363 return bytes2byteArray(bytes, bytes.length, separator); 364 } 365 366 /** 367 * Splits first len bytes in bytes to array of arrays of bytes 368 * on byte separator 369 * @param bytes the byte array to split 370 * @param len the number of bytes to split 371 * @param separator the delimiting byte 372 */ 373 public static byte[][] bytes2byteArray(byte[] bytes, 374 int len, 375 byte separator) { 376 assert len <= bytes.length; 377 int splits = 0; 378 if (len == 0) { 379 return new byte[][]{null}; 380 } 381 // Count the splits. Omit multiple separators and the last one 382 for (int i = 0; i < len; i++) { 383 if (bytes[i] == separator) { 384 splits++; 385 } 386 } 387 int last = len - 1; 388 while (last > -1 && bytes[last--] == separator) { 389 splits--; 390 } 391 if (splits == 0 && bytes[0] == separator) { 392 return new byte[][]{null}; 393 } 394 splits++; 395 byte[][] result = new byte[splits][]; 396 int startIndex = 0; 397 int nextIndex = 0; 398 int index = 0; 399 // Build the splits 400 while (index < splits) { 401 while (nextIndex < len && bytes[nextIndex] != separator) { 402 nextIndex++; 403 } 404 result[index] = new byte[nextIndex - startIndex]; 405 System.arraycopy(bytes, startIndex, result[index], 0, nextIndex 406 - startIndex); 407 index++; 408 startIndex = nextIndex + 1; 409 nextIndex = startIndex; 410 } 411 return result; 412 } 413 414 /** 415 * Convert a LocatedBlocks to BlockLocations[] 416 * @param blocks a LocatedBlocks 417 * @return an array of BlockLocations 418 */ 419 public static BlockLocation[] locatedBlocks2Locations(LocatedBlocks blocks) { 420 if (blocks == null) { 421 return new BlockLocation[0]; 422 } 423 return locatedBlocks2Locations(blocks.getLocatedBlocks()); 424 } 425 426 /** 427 * Convert a List<LocatedBlock> to BlockLocation[] 428 * @param blocks A List<LocatedBlock> to be converted 429 * @return converted array of BlockLocation 430 */ 431 public static BlockLocation[] locatedBlocks2Locations(List<LocatedBlock> blocks) { 432 if (blocks == null) { 433 return new BlockLocation[0]; 434 } 435 int nrBlocks = blocks.size(); 436 BlockLocation[] blkLocations = new BlockLocation[nrBlocks]; 437 if (nrBlocks == 0) { 438 return blkLocations; 439 } 440 int idx = 0; 441 for (LocatedBlock blk : blocks) { 442 assert idx < nrBlocks : "Incorrect index"; 443 DatanodeInfo[] locations = blk.getLocations(); 444 String[] hosts = new String[locations.length]; 445 String[] xferAddrs = new String[locations.length]; 446 String[] racks = new String[locations.length]; 447 for (int hCnt = 0; hCnt < locations.length; hCnt++) { 448 hosts[hCnt] = locations[hCnt].getHostName(); 449 xferAddrs[hCnt] = locations[hCnt].getXferAddr(); 450 NodeBase node = new NodeBase(xferAddrs[hCnt], 451 locations[hCnt].getNetworkLocation()); 452 racks[hCnt] = node.toString(); 453 } 454 DatanodeInfo[] cachedLocations = blk.getCachedLocations(); 455 String[] cachedHosts = new String[cachedLocations.length]; 456 for (int i=0; i<cachedLocations.length; i++) { 457 cachedHosts[i] = cachedLocations[i].getHostName(); 458 } 459 blkLocations[idx] = new BlockLocation(xferAddrs, hosts, cachedHosts, 460 racks, 461 blk.getStartOffset(), 462 blk.getBlockSize(), 463 blk.isCorrupt()); 464 idx++; 465 } 466 return blkLocations; 467 } 468 469 /** 470 * Returns collection of nameservice Ids from the configuration. 471 * @param conf configuration 472 * @return collection of nameservice Ids, or null if not specified 473 */ 474 public static Collection<String> getNameServiceIds(Configuration conf) { 475 return conf.getTrimmedStringCollection(DFS_NAMESERVICES); 476 } 477 478 /** 479 * @return <code>coll</code> if it is non-null and non-empty. Otherwise, 480 * returns a list with a single null value. 481 */ 482 private static Collection<String> emptyAsSingletonNull(Collection<String> coll) { 483 if (coll == null || coll.isEmpty()) { 484 return Collections.singletonList(null); 485 } else { 486 return coll; 487 } 488 } 489 490 /** 491 * Namenode HighAvailability related configuration. 492 * Returns collection of namenode Ids from the configuration. One logical id 493 * for each namenode in the in the HA setup. 494 * 495 * @param conf configuration 496 * @param nsId the nameservice ID to look at, or null for non-federated 497 * @return collection of namenode Ids 498 */ 499 public static Collection<String> getNameNodeIds(Configuration conf, String nsId) { 500 String key = addSuffix(DFS_HA_NAMENODES_KEY_PREFIX, nsId); 501 return conf.getTrimmedStringCollection(key); 502 } 503 504 /** 505 * Given a list of keys in the order of preference, returns a value 506 * for the key in the given order from the configuration. 507 * @param defaultValue default value to return, when key was not found 508 * @param keySuffix suffix to add to the key, if it is not null 509 * @param conf Configuration 510 * @param keys list of keys in the order of preference 511 * @return value of the key or default if a key was not found in configuration 512 */ 513 private static String getConfValue(String defaultValue, String keySuffix, 514 Configuration conf, String... keys) { 515 String value = null; 516 for (String key : keys) { 517 key = addSuffix(key, keySuffix); 518 value = conf.get(key); 519 if (value != null) { 520 break; 521 } 522 } 523 if (value == null) { 524 value = defaultValue; 525 } 526 return value; 527 } 528 529 /** Add non empty and non null suffix to a key */ 530 private static String addSuffix(String key, String suffix) { 531 if (suffix == null || suffix.isEmpty()) { 532 return key; 533 } 534 assert !suffix.startsWith(".") : 535 "suffix '" + suffix + "' should not already have '.' prepended."; 536 return key + "." + suffix; 537 } 538 539 /** Concatenate list of suffix strings '.' separated */ 540 private static String concatSuffixes(String... suffixes) { 541 if (suffixes == null) { 542 return null; 543 } 544 return Joiner.on(".").skipNulls().join(suffixes); 545 } 546 547 /** 548 * Return configuration key of format key.suffix1.suffix2...suffixN 549 */ 550 public static String addKeySuffixes(String key, String... suffixes) { 551 String keySuffix = concatSuffixes(suffixes); 552 return addSuffix(key, keySuffix); 553 } 554 555 /** 556 * Returns the configured address for all NameNodes in the cluster. 557 * @param conf configuration 558 * @param defaultAddress default address to return in case key is not found. 559 * @param keys Set of keys to look for in the order of preference 560 * @return a map(nameserviceId to map(namenodeId to InetSocketAddress)) 561 */ 562 private static Map<String, Map<String, InetSocketAddress>> 563 getAddresses(Configuration conf, 564 String defaultAddress, String... keys) { 565 Collection<String> nameserviceIds = getNameServiceIds(conf); 566 567 // Look for configurations of the form <key>[.<nameserviceId>][.<namenodeId>] 568 // across all of the configured nameservices and namenodes. 569 Map<String, Map<String, InetSocketAddress>> ret = Maps.newLinkedHashMap(); 570 for (String nsId : emptyAsSingletonNull(nameserviceIds)) { 571 Map<String, InetSocketAddress> isas = 572 getAddressesForNameserviceId(conf, nsId, defaultAddress, keys); 573 if (!isas.isEmpty()) { 574 ret.put(nsId, isas); 575 } 576 } 577 return ret; 578 } 579 580 private static Map<String, InetSocketAddress> getAddressesForNameserviceId( 581 Configuration conf, String nsId, String defaultValue, 582 String[] keys) { 583 Collection<String> nnIds = getNameNodeIds(conf, nsId); 584 Map<String, InetSocketAddress> ret = Maps.newHashMap(); 585 for (String nnId : emptyAsSingletonNull(nnIds)) { 586 String suffix = concatSuffixes(nsId, nnId); 587 String address = getConfValue(defaultValue, suffix, conf, keys); 588 if (address != null) { 589 InetSocketAddress isa = NetUtils.createSocketAddr(address); 590 if (isa.isUnresolved()) { 591 LOG.warn("Namenode for " + nsId + 592 " remains unresolved for ID " + nnId + 593 ". Check your hdfs-site.xml file to " + 594 "ensure namenodes are configured properly."); 595 } 596 ret.put(nnId, isa); 597 } 598 } 599 return ret; 600 } 601 602 /** 603 * @return a collection of all configured NN Kerberos principals. 604 */ 605 public static Set<String> getAllNnPrincipals(Configuration conf) throws IOException { 606 Set<String> principals = new HashSet<String>(); 607 for (String nsId : DFSUtil.getNameServiceIds(conf)) { 608 if (HAUtil.isHAEnabled(conf, nsId)) { 609 for (String nnId : DFSUtil.getNameNodeIds(conf, nsId)) { 610 Configuration confForNn = new Configuration(conf); 611 NameNode.initializeGenericKeys(confForNn, nsId, nnId); 612 String principal = SecurityUtil.getServerPrincipal(confForNn 613 .get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY), 614 NameNode.getAddress(confForNn).getHostName()); 615 principals.add(principal); 616 } 617 } else { 618 Configuration confForNn = new Configuration(conf); 619 NameNode.initializeGenericKeys(confForNn, nsId, null); 620 String principal = SecurityUtil.getServerPrincipal(confForNn 621 .get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY), 622 NameNode.getAddress(confForNn).getHostName()); 623 principals.add(principal); 624 } 625 } 626 627 return principals; 628 } 629 630 /** 631 * Returns list of InetSocketAddress corresponding to HA NN RPC addresses from 632 * the configuration. 633 * 634 * @param conf configuration 635 * @return list of InetSocketAddresses 636 */ 637 public static Map<String, Map<String, InetSocketAddress>> getHaNnRpcAddresses( 638 Configuration conf) { 639 return getAddresses(conf, null, DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY); 640 } 641 642 /** 643 * Returns list of InetSocketAddress corresponding to HA NN HTTP addresses from 644 * the configuration. 645 * 646 * @return list of InetSocketAddresses 647 */ 648 public static Map<String, Map<String, InetSocketAddress>> getHaNnWebHdfsAddresses( 649 Configuration conf, String scheme) { 650 if (WebHdfsFileSystem.SCHEME.equals(scheme)) { 651 return getAddresses(conf, null, 652 DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY); 653 } else if (SWebHdfsFileSystem.SCHEME.equals(scheme)) { 654 return getAddresses(conf, null, 655 DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY); 656 } else { 657 throw new IllegalArgumentException("Unsupported scheme: " + scheme); 658 } 659 } 660 661 /** 662 * Resolve an HDFS URL into real INetSocketAddress. It works like a DNS resolver 663 * when the URL points to an non-HA cluster. When the URL points to an HA 664 * cluster, the resolver further resolves the logical name (i.e., the authority 665 * in the URL) into real namenode addresses. 666 */ 667 public static InetSocketAddress[] resolveWebHdfsUri(URI uri, Configuration conf) 668 throws IOException { 669 int defaultPort; 670 String scheme = uri.getScheme(); 671 if (WebHdfsFileSystem.SCHEME.equals(scheme)) { 672 defaultPort = DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT; 673 } else if (SWebHdfsFileSystem.SCHEME.equals(scheme)) { 674 defaultPort = DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT; 675 } else { 676 throw new IllegalArgumentException("Unsupported scheme: " + scheme); 677 } 678 679 ArrayList<InetSocketAddress> ret = new ArrayList<InetSocketAddress>(); 680 681 if (!HAUtil.isLogicalUri(conf, uri)) { 682 InetSocketAddress addr = NetUtils.createSocketAddr(uri.getAuthority(), 683 defaultPort); 684 ret.add(addr); 685 686 } else { 687 Map<String, Map<String, InetSocketAddress>> addresses = DFSUtil 688 .getHaNnWebHdfsAddresses(conf, scheme); 689 690 for (Map<String, InetSocketAddress> addrs : addresses.values()) { 691 for (InetSocketAddress addr : addrs.values()) { 692 ret.add(addr); 693 } 694 } 695 } 696 697 InetSocketAddress[] r = new InetSocketAddress[ret.size()]; 698 return ret.toArray(r); 699 } 700 701 /** 702 * Returns list of InetSocketAddress corresponding to backup node rpc 703 * addresses from the configuration. 704 * 705 * @param conf configuration 706 * @return list of InetSocketAddresses 707 * @throws IOException on error 708 */ 709 public static Map<String, Map<String, InetSocketAddress>> getBackupNodeAddresses( 710 Configuration conf) throws IOException { 711 Map<String, Map<String, InetSocketAddress>> addressList = getAddresses(conf, 712 null, DFS_NAMENODE_BACKUP_ADDRESS_KEY); 713 if (addressList.isEmpty()) { 714 throw new IOException("Incorrect configuration: backup node address " 715 + DFS_NAMENODE_BACKUP_ADDRESS_KEY + " is not configured."); 716 } 717 return addressList; 718 } 719 720 /** 721 * Returns list of InetSocketAddresses of corresponding to secondary namenode 722 * http addresses from the configuration. 723 * 724 * @param conf configuration 725 * @return list of InetSocketAddresses 726 * @throws IOException on error 727 */ 728 public static Map<String, Map<String, InetSocketAddress>> getSecondaryNameNodeAddresses( 729 Configuration conf) throws IOException { 730 Map<String, Map<String, InetSocketAddress>> addressList = getAddresses(conf, null, 731 DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY); 732 if (addressList.isEmpty()) { 733 throw new IOException("Incorrect configuration: secondary namenode address " 734 + DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY + " is not configured."); 735 } 736 return addressList; 737 } 738 739 /** 740 * Returns list of InetSocketAddresses corresponding to namenodes from the 741 * configuration. Note this is to be used by datanodes to get the list of 742 * namenode addresses to talk to. 743 * 744 * Returns namenode address specifically configured for datanodes (using 745 * service ports), if found. If not, regular RPC address configured for other 746 * clients is returned. 747 * 748 * @param conf configuration 749 * @return list of InetSocketAddress 750 * @throws IOException on error 751 */ 752 public static Map<String, Map<String, InetSocketAddress>> getNNServiceRpcAddresses( 753 Configuration conf) throws IOException { 754 // Use default address as fall back 755 String defaultAddress; 756 try { 757 defaultAddress = NetUtils.getHostPortString(NameNode.getAddress(conf)); 758 } catch (IllegalArgumentException e) { 759 defaultAddress = null; 760 } 761 762 Map<String, Map<String, InetSocketAddress>> addressList = 763 getAddresses(conf, defaultAddress, 764 DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY); 765 if (addressList.isEmpty()) { 766 throw new IOException("Incorrect configuration: namenode address " 767 + DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY + " or " 768 + DFS_NAMENODE_RPC_ADDRESS_KEY 769 + " is not configured."); 770 } 771 return addressList; 772 } 773 774 /** 775 * Flatten the given map, as returned by other functions in this class, 776 * into a flat list of {@link ConfiguredNNAddress} instances. 777 */ 778 public static List<ConfiguredNNAddress> flattenAddressMap( 779 Map<String, Map<String, InetSocketAddress>> map) { 780 List<ConfiguredNNAddress> ret = Lists.newArrayList(); 781 782 for (Map.Entry<String, Map<String, InetSocketAddress>> entry : 783 map.entrySet()) { 784 String nsId = entry.getKey(); 785 Map<String, InetSocketAddress> nnMap = entry.getValue(); 786 for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) { 787 String nnId = e2.getKey(); 788 InetSocketAddress addr = e2.getValue(); 789 790 ret.add(new ConfiguredNNAddress(nsId, nnId, addr)); 791 } 792 } 793 return ret; 794 } 795 796 /** 797 * Format the given map, as returned by other functions in this class, 798 * into a string suitable for debugging display. The format of this string 799 * should not be considered an interface, and is liable to change. 800 */ 801 public static String addressMapToString( 802 Map<String, Map<String, InetSocketAddress>> map) { 803 StringBuilder b = new StringBuilder(); 804 for (Map.Entry<String, Map<String, InetSocketAddress>> entry : 805 map.entrySet()) { 806 String nsId = entry.getKey(); 807 Map<String, InetSocketAddress> nnMap = entry.getValue(); 808 b.append("Nameservice <").append(nsId).append(">:").append("\n"); 809 for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) { 810 b.append(" NN ID ").append(e2.getKey()) 811 .append(" => ").append(e2.getValue()).append("\n"); 812 } 813 } 814 return b.toString(); 815 } 816 817 public static String nnAddressesAsString(Configuration conf) { 818 Map<String, Map<String, InetSocketAddress>> addresses = 819 getHaNnRpcAddresses(conf); 820 return addressMapToString(addresses); 821 } 822 823 /** 824 * Represent one of the NameNodes configured in the cluster. 825 */ 826 public static class ConfiguredNNAddress { 827 private final String nameserviceId; 828 private final String namenodeId; 829 private final InetSocketAddress addr; 830 831 private ConfiguredNNAddress(String nameserviceId, String namenodeId, 832 InetSocketAddress addr) { 833 this.nameserviceId = nameserviceId; 834 this.namenodeId = namenodeId; 835 this.addr = addr; 836 } 837 838 public String getNameserviceId() { 839 return nameserviceId; 840 } 841 842 public String getNamenodeId() { 843 return namenodeId; 844 } 845 846 public InetSocketAddress getAddress() { 847 return addr; 848 } 849 850 @Override 851 public String toString() { 852 return "ConfiguredNNAddress[nsId=" + nameserviceId + ";" + 853 "nnId=" + namenodeId + ";addr=" + addr + "]"; 854 } 855 } 856 857 /** 858 * Get a URI for each configured nameservice. If a nameservice is 859 * HA-enabled, then the logical URI of the nameservice is returned. If the 860 * nameservice is not HA-enabled, then a URI corresponding to an RPC address 861 * of the single NN for that nameservice is returned, preferring the service 862 * RPC address over the client RPC address. 863 * 864 * @param conf configuration 865 * @return a collection of all configured NN URIs, preferring service 866 * addresses 867 */ 868 public static Collection<URI> getNsServiceRpcUris(Configuration conf) { 869 return getNameServiceUris(conf, 870 DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, 871 DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY); 872 } 873 874 /** 875 * Get a URI for each configured nameservice. If a nameservice is 876 * HA-enabled, then the logical URI of the nameservice is returned. If the 877 * nameservice is not HA-enabled, then a URI corresponding to the address of 878 * the single NN for that nameservice is returned. 879 * 880 * @param conf configuration 881 * @param keys configuration keys to try in order to get the URI for non-HA 882 * nameservices 883 * @return a collection of all configured NN URIs 884 */ 885 public static Collection<URI> getNameServiceUris(Configuration conf, 886 String... keys) { 887 Set<URI> ret = new HashSet<URI>(); 888 889 // We're passed multiple possible configuration keys for any given NN or HA 890 // nameservice, and search the config in order of these keys. In order to 891 // make sure that a later config lookup (e.g. fs.defaultFS) doesn't add a 892 // URI for a config key for which we've already found a preferred entry, we 893 // keep track of non-preferred keys here. 894 Set<URI> nonPreferredUris = new HashSet<URI>(); 895 896 for (String nsId : getNameServiceIds(conf)) { 897 if (HAUtil.isHAEnabled(conf, nsId)) { 898 // Add the logical URI of the nameservice. 899 try { 900 ret.add(new URI(HdfsConstants.HDFS_URI_SCHEME + "://" + nsId)); 901 } catch (URISyntaxException ue) { 902 throw new IllegalArgumentException(ue); 903 } 904 } else { 905 // Add the URI corresponding to the address of the NN. 906 boolean uriFound = false; 907 for (String key : keys) { 908 String addr = conf.get(concatSuffixes(key, nsId)); 909 if (addr != null) { 910 URI uri = createUri(HdfsConstants.HDFS_URI_SCHEME, 911 NetUtils.createSocketAddr(addr)); 912 if (!uriFound) { 913 uriFound = true; 914 ret.add(uri); 915 } else { 916 nonPreferredUris.add(uri); 917 } 918 } 919 } 920 } 921 } 922 923 // Add the generic configuration keys. 924 boolean uriFound = false; 925 for (String key : keys) { 926 String addr = conf.get(key); 927 if (addr != null) { 928 URI uri = createUri("hdfs", NetUtils.createSocketAddr(addr)); 929 if (!uriFound) { 930 uriFound = true; 931 ret.add(uri); 932 } else { 933 nonPreferredUris.add(uri); 934 } 935 } 936 } 937 938 // Add the default URI if it is an HDFS URI. 939 URI defaultUri = FileSystem.getDefaultUri(conf); 940 // checks if defaultUri is ip:port format 941 // and convert it to hostname:port format 942 if (defaultUri != null && (defaultUri.getPort() != -1)) { 943 defaultUri = createUri(defaultUri.getScheme(), 944 NetUtils.createSocketAddr(defaultUri.getHost(), 945 defaultUri.getPort())); 946 } 947 if (defaultUri != null && 948 HdfsConstants.HDFS_URI_SCHEME.equals(defaultUri.getScheme()) && 949 !nonPreferredUris.contains(defaultUri)) { 950 ret.add(defaultUri); 951 } 952 953 return ret; 954 } 955 956 /** 957 * Given the InetSocketAddress this method returns the nameservice Id 958 * corresponding to the key with matching address, by doing a reverse 959 * lookup on the list of nameservices until it finds a match. 960 * 961 * Since the process of resolving URIs to Addresses is slightly expensive, 962 * this utility method should not be used in performance-critical routines. 963 * 964 * @param conf - configuration 965 * @param address - InetSocketAddress for configured communication with NN. 966 * Configured addresses are typically given as URIs, but we may have to 967 * compare against a URI typed in by a human, or the server name may be 968 * aliased, so we compare unambiguous InetSocketAddresses instead of just 969 * comparing URI substrings. 970 * @param keys - list of configured communication parameters that should 971 * be checked for matches. For example, to compare against RPC addresses, 972 * provide the list DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, 973 * DFS_NAMENODE_RPC_ADDRESS_KEY. Use the generic parameter keys, 974 * not the NameServiceId-suffixed keys. 975 * @return nameserviceId, or null if no match found 976 */ 977 public static String getNameServiceIdFromAddress(final Configuration conf, 978 final InetSocketAddress address, String... keys) { 979 // Configuration with a single namenode and no nameserviceId 980 String[] ids = getSuffixIDs(conf, address, keys); 981 return (ids != null) ? ids[0] : null; 982 } 983 984 /** 985 * return server http or https address from the configuration for a 986 * given namenode rpc address. 987 * @param conf 988 * @param namenodeAddr - namenode RPC address 989 * @param scheme - the scheme (http / https) 990 * @return server http or https address 991 * @throws IOException 992 */ 993 public static URI getInfoServer(InetSocketAddress namenodeAddr, 994 Configuration conf, String scheme) throws IOException { 995 String[] suffixes = null; 996 if (namenodeAddr != null) { 997 // if non-default namenode, try reverse look up 998 // the nameServiceID if it is available 999 suffixes = getSuffixIDs(conf, namenodeAddr, 1000 DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, 1001 DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY); 1002 } 1003 1004 String authority; 1005 if ("http".equals(scheme)) { 1006 authority = getSuffixedConf(conf, DFS_NAMENODE_HTTP_ADDRESS_KEY, 1007 DFS_NAMENODE_HTTP_ADDRESS_DEFAULT, suffixes); 1008 } else if ("https".equals(scheme)) { 1009 authority = getSuffixedConf(conf, DFS_NAMENODE_HTTPS_ADDRESS_KEY, 1010 DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT, suffixes); 1011 } else { 1012 throw new IllegalArgumentException("Invalid scheme:" + scheme); 1013 } 1014 1015 if (namenodeAddr != null) { 1016 authority = substituteForWildcardAddress(authority, 1017 namenodeAddr.getHostName()); 1018 } 1019 return URI.create(scheme + "://" + authority); 1020 } 1021 1022 /** 1023 * Lookup the HTTP / HTTPS address of the namenode, and replace its hostname 1024 * with defaultHost when it found out that the address is a wildcard / local 1025 * address. 1026 * 1027 * @param defaultHost 1028 * The default host name of the namenode. 1029 * @param conf 1030 * The configuration 1031 * @param scheme 1032 * HTTP or HTTPS 1033 * @throws IOException 1034 */ 1035 public static URI getInfoServerWithDefaultHost(String defaultHost, 1036 Configuration conf, final String scheme) throws IOException { 1037 URI configuredAddr = getInfoServer(null, conf, scheme); 1038 String authority = substituteForWildcardAddress( 1039 configuredAddr.getAuthority(), defaultHost); 1040 return URI.create(scheme + "://" + authority); 1041 } 1042 1043 /** 1044 * Determine whether HTTP or HTTPS should be used to connect to the remote 1045 * server. Currently the client only connects to the server via HTTPS if the 1046 * policy is set to HTTPS_ONLY. 1047 * 1048 * @return the scheme (HTTP / HTTPS) 1049 */ 1050 public static String getHttpClientScheme(Configuration conf) { 1051 HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf); 1052 return policy == HttpConfig.Policy.HTTPS_ONLY ? "https" : "http"; 1053 } 1054 1055 /** 1056 * Substitute a default host in the case that an address has been configured 1057 * with a wildcard. This is used, for example, when determining the HTTP 1058 * address of the NN -- if it's configured to bind to 0.0.0.0, we want to 1059 * substitute the hostname from the filesystem URI rather than trying to 1060 * connect to 0.0.0.0. 1061 * @param configuredAddress the address found in the configuration 1062 * @param defaultHost the host to substitute with, if configuredAddress 1063 * is a local/wildcard address. 1064 * @return the substituted address 1065 * @throws IOException if it is a wildcard address and security is enabled 1066 */ 1067 @VisibleForTesting 1068 static String substituteForWildcardAddress(String configuredAddress, 1069 String defaultHost) throws IOException { 1070 InetSocketAddress sockAddr = NetUtils.createSocketAddr(configuredAddress); 1071 InetSocketAddress defaultSockAddr = NetUtils.createSocketAddr(defaultHost 1072 + ":0"); 1073 if (sockAddr.getAddress().isAnyLocalAddress()) { 1074 if (UserGroupInformation.isSecurityEnabled() && 1075 defaultSockAddr.getAddress().isAnyLocalAddress()) { 1076 throw new IOException("Cannot use a wildcard address with security. " + 1077 "Must explicitly set bind address for Kerberos"); 1078 } 1079 return defaultHost + ":" + sockAddr.getPort(); 1080 } else { 1081 return configuredAddress; 1082 } 1083 } 1084 1085 private static String getSuffixedConf(Configuration conf, 1086 String key, String defaultVal, String[] suffixes) { 1087 String ret = conf.get(DFSUtil.addKeySuffixes(key, suffixes)); 1088 if (ret != null) { 1089 return ret; 1090 } 1091 return conf.get(key, defaultVal); 1092 } 1093 1094 /** 1095 * Sets the node specific setting into generic configuration key. Looks up 1096 * value of "key.nameserviceId.namenodeId" and if found sets that value into 1097 * generic key in the conf. If this is not found, falls back to 1098 * "key.nameserviceId" and then the unmodified key. 1099 * 1100 * Note that this only modifies the runtime conf. 1101 * 1102 * @param conf 1103 * Configuration object to lookup specific key and to set the value 1104 * to the key passed. Note the conf object is modified. 1105 * @param nameserviceId 1106 * nameservice Id to construct the node specific key. Pass null if 1107 * federation is not configuration. 1108 * @param nnId 1109 * namenode Id to construct the node specific key. Pass null if 1110 * HA is not configured. 1111 * @param keys 1112 * The key for which node specific value is looked up 1113 */ 1114 public static void setGenericConf(Configuration conf, 1115 String nameserviceId, String nnId, String... keys) { 1116 for (String key : keys) { 1117 String value = conf.get(addKeySuffixes(key, nameserviceId, nnId)); 1118 if (value != null) { 1119 conf.set(key, value); 1120 continue; 1121 } 1122 value = conf.get(addKeySuffixes(key, nameserviceId)); 1123 if (value != null) { 1124 conf.set(key, value); 1125 } 1126 } 1127 } 1128 1129 /** Return used as percentage of capacity */ 1130 public static float getPercentUsed(long used, long capacity) { 1131 return capacity <= 0 ? 100 : (used * 100.0f)/capacity; 1132 } 1133 1134 /** Return remaining as percentage of capacity */ 1135 public static float getPercentRemaining(long remaining, long capacity) { 1136 return capacity <= 0 ? 0 : (remaining * 100.0f)/capacity; 1137 } 1138 1139 /** Convert percentage to a string. */ 1140 public static String percent2String(double percentage) { 1141 return StringUtils.format("%.2f%%", percentage); 1142 } 1143 1144 /** 1145 * Round bytes to GiB (gibibyte) 1146 * @param bytes number of bytes 1147 * @return number of GiB 1148 */ 1149 public static int roundBytesToGB(long bytes) { 1150 return Math.round((float)bytes/ 1024 / 1024 / 1024); 1151 } 1152 1153 /** Create a {@link ClientDatanodeProtocol} proxy */ 1154 public static ClientDatanodeProtocol createClientDatanodeProtocolProxy( 1155 DatanodeID datanodeid, Configuration conf, int socketTimeout, 1156 boolean connectToDnViaHostname, LocatedBlock locatedBlock) throws IOException { 1157 return new ClientDatanodeProtocolTranslatorPB(datanodeid, conf, socketTimeout, 1158 connectToDnViaHostname, locatedBlock); 1159 } 1160 1161 /** Create {@link ClientDatanodeProtocol} proxy using kerberos ticket */ 1162 static ClientDatanodeProtocol createClientDatanodeProtocolProxy( 1163 DatanodeID datanodeid, Configuration conf, int socketTimeout, 1164 boolean connectToDnViaHostname) throws IOException { 1165 return new ClientDatanodeProtocolTranslatorPB( 1166 datanodeid, conf, socketTimeout, connectToDnViaHostname); 1167 } 1168 1169 /** Create a {@link ClientDatanodeProtocol} proxy */ 1170 public static ClientDatanodeProtocol createClientDatanodeProtocolProxy( 1171 InetSocketAddress addr, UserGroupInformation ticket, Configuration conf, 1172 SocketFactory factory) throws IOException { 1173 return new ClientDatanodeProtocolTranslatorPB(addr, ticket, conf, factory); 1174 } 1175 1176 /** 1177 * Get nameservice Id for the {@link NameNode} based on namenode RPC address 1178 * matching the local node address. 1179 */ 1180 public static String getNamenodeNameServiceId(Configuration conf) { 1181 return getNameServiceId(conf, DFS_NAMENODE_RPC_ADDRESS_KEY); 1182 } 1183 1184 /** 1185 * Get nameservice Id for the BackupNode based on backup node RPC address 1186 * matching the local node address. 1187 */ 1188 public static String getBackupNameServiceId(Configuration conf) { 1189 return getNameServiceId(conf, DFS_NAMENODE_BACKUP_ADDRESS_KEY); 1190 } 1191 1192 /** 1193 * Get nameservice Id for the secondary node based on secondary http address 1194 * matching the local node address. 1195 */ 1196 public static String getSecondaryNameServiceId(Configuration conf) { 1197 return getNameServiceId(conf, DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY); 1198 } 1199 1200 /** 1201 * Get the nameservice Id by matching the {@code addressKey} with the 1202 * the address of the local node. 1203 * 1204 * If {@link DFSConfigKeys#DFS_NAMESERVICE_ID} is not specifically 1205 * configured, and more than one nameservice Id is configured, this method 1206 * determines the nameservice Id by matching the local node's address with the 1207 * configured addresses. When a match is found, it returns the nameservice Id 1208 * from the corresponding configuration key. 1209 * 1210 * @param conf Configuration 1211 * @param addressKey configuration key to get the address. 1212 * @return nameservice Id on success, null if federation is not configured. 1213 * @throws HadoopIllegalArgumentException on error 1214 */ 1215 private static String getNameServiceId(Configuration conf, String addressKey) { 1216 String nameserviceId = conf.get(DFS_NAMESERVICE_ID); 1217 if (nameserviceId != null) { 1218 return nameserviceId; 1219 } 1220 Collection<String> nsIds = getNameServiceIds(conf); 1221 if (1 == nsIds.size()) { 1222 return nsIds.toArray(new String[1])[0]; 1223 } 1224 String nnId = conf.get(DFS_HA_NAMENODE_ID_KEY); 1225 1226 return getSuffixIDs(conf, addressKey, null, nnId, LOCAL_ADDRESS_MATCHER)[0]; 1227 } 1228 1229 /** 1230 * Returns nameservice Id and namenode Id when the local host matches the 1231 * configuration parameter {@code addressKey}.<nameservice Id>.<namenode Id> 1232 * 1233 * @param conf Configuration 1234 * @param addressKey configuration key corresponding to the address. 1235 * @param knownNsId only look at configs for the given nameservice, if not-null 1236 * @param knownNNId only look at configs for the given namenode, if not null 1237 * @param matcher matching criteria for matching the address 1238 * @return Array with nameservice Id and namenode Id on success. First element 1239 * in the array is nameservice Id and second element is namenode Id. 1240 * Null value indicates that the configuration does not have the the 1241 * Id. 1242 * @throws HadoopIllegalArgumentException on error 1243 */ 1244 static String[] getSuffixIDs(final Configuration conf, final String addressKey, 1245 String knownNsId, String knownNNId, 1246 final AddressMatcher matcher) { 1247 String nameserviceId = null; 1248 String namenodeId = null; 1249 int found = 0; 1250 1251 Collection<String> nsIds = getNameServiceIds(conf); 1252 for (String nsId : emptyAsSingletonNull(nsIds)) { 1253 if (knownNsId != null && !knownNsId.equals(nsId)) { 1254 continue; 1255 } 1256 1257 Collection<String> nnIds = getNameNodeIds(conf, nsId); 1258 for (String nnId : emptyAsSingletonNull(nnIds)) { 1259 if (LOG.isTraceEnabled()) { 1260 LOG.trace(String.format("addressKey: %s nsId: %s nnId: %s", 1261 addressKey, nsId, nnId)); 1262 } 1263 if (knownNNId != null && !knownNNId.equals(nnId)) { 1264 continue; 1265 } 1266 String key = addKeySuffixes(addressKey, nsId, nnId); 1267 String addr = conf.get(key); 1268 if (addr == null) { 1269 continue; 1270 } 1271 InetSocketAddress s = null; 1272 try { 1273 s = NetUtils.createSocketAddr(addr); 1274 } catch (Exception e) { 1275 LOG.warn("Exception in creating socket address " + addr, e); 1276 continue; 1277 } 1278 if (!s.isUnresolved() && matcher.match(s)) { 1279 nameserviceId = nsId; 1280 namenodeId = nnId; 1281 found++; 1282 } 1283 } 1284 } 1285 if (found > 1) { // Only one address must match the local address 1286 String msg = "Configuration has multiple addresses that match " 1287 + "local node's address. Please configure the system with " 1288 + DFS_NAMESERVICE_ID + " and " 1289 + DFS_HA_NAMENODE_ID_KEY; 1290 throw new HadoopIllegalArgumentException(msg); 1291 } 1292 return new String[] { nameserviceId, namenodeId }; 1293 } 1294 1295 /** 1296 * For given set of {@code keys} adds nameservice Id and or namenode Id 1297 * and returns {nameserviceId, namenodeId} when address match is found. 1298 * @see #getSuffixIDs(Configuration, String, AddressMatcher) 1299 */ 1300 static String[] getSuffixIDs(final Configuration conf, 1301 final InetSocketAddress address, final String... keys) { 1302 AddressMatcher matcher = new AddressMatcher() { 1303 @Override 1304 public boolean match(InetSocketAddress s) { 1305 return address.equals(s); 1306 } 1307 }; 1308 1309 for (String key : keys) { 1310 String[] ids = getSuffixIDs(conf, key, null, null, matcher); 1311 if (ids != null && (ids [0] != null || ids[1] != null)) { 1312 return ids; 1313 } 1314 } 1315 return null; 1316 } 1317 1318 private interface AddressMatcher { 1319 public boolean match(InetSocketAddress s); 1320 } 1321 1322 /** Create a URI from the scheme and address */ 1323 public static URI createUri(String scheme, InetSocketAddress address) { 1324 try { 1325 return new URI(scheme, null, address.getHostName(), address.getPort(), 1326 null, null, null); 1327 } catch (URISyntaxException ue) { 1328 throw new IllegalArgumentException(ue); 1329 } 1330 } 1331 1332 /** 1333 * Add protobuf based protocol to the {@link org.apache.hadoop.ipc.RPC.Server} 1334 * @param conf configuration 1335 * @param protocol Protocol interface 1336 * @param service service that implements the protocol 1337 * @param server RPC server to which the protocol & implementation is added to 1338 * @throws IOException 1339 */ 1340 public static void addPBProtocol(Configuration conf, Class<?> protocol, 1341 BlockingService service, RPC.Server server) throws IOException { 1342 RPC.setProtocolEngine(conf, protocol, ProtobufRpcEngine.class); 1343 server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, protocol, service); 1344 } 1345 1346 /** 1347 * Map a logical namenode ID to its service address. Use the given 1348 * nameservice if specified, or the configured one if none is given. 1349 * 1350 * @param conf Configuration 1351 * @param nsId which nameservice nnId is a part of, optional 1352 * @param nnId the namenode ID to get the service addr for 1353 * @return the service addr, null if it could not be determined 1354 */ 1355 public static String getNamenodeServiceAddr(final Configuration conf, 1356 String nsId, String nnId) { 1357 1358 if (nsId == null) { 1359 nsId = getOnlyNameServiceIdOrNull(conf); 1360 } 1361 1362 String serviceAddrKey = concatSuffixes( 1363 DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, nsId, nnId); 1364 1365 String addrKey = concatSuffixes( 1366 DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, nsId, nnId); 1367 1368 String serviceRpcAddr = conf.get(serviceAddrKey); 1369 if (serviceRpcAddr == null) { 1370 serviceRpcAddr = conf.get(addrKey); 1371 } 1372 return serviceRpcAddr; 1373 } 1374 1375 /** 1376 * If the configuration refers to only a single nameservice, return the 1377 * name of that nameservice. If it refers to 0 or more than 1, return null. 1378 */ 1379 public static String getOnlyNameServiceIdOrNull(Configuration conf) { 1380 Collection<String> nsIds = getNameServiceIds(conf); 1381 if (1 == nsIds.size()) { 1382 return nsIds.toArray(new String[1])[0]; 1383 } else { 1384 // No nameservice ID was given and more than one is configured 1385 return null; 1386 } 1387 } 1388 1389 public static Options helpOptions = new Options(); 1390 public static Option helpOpt = new Option("h", "help", false, 1391 "get help information"); 1392 1393 static { 1394 helpOptions.addOption(helpOpt); 1395 } 1396 1397 /** 1398 * Parse the arguments for commands 1399 * 1400 * @param args the argument to be parsed 1401 * @param helpDescription help information to be printed out 1402 * @param out Printer 1403 * @param printGenericCommandUsage whether to print the 1404 * generic command usage defined in ToolRunner 1405 * @return true when the argument matches help option, false if not 1406 */ 1407 public static boolean parseHelpArgument(String[] args, 1408 String helpDescription, PrintStream out, boolean printGenericCommandUsage) { 1409 if (args.length == 1) { 1410 try { 1411 CommandLineParser parser = new PosixParser(); 1412 CommandLine cmdLine = parser.parse(helpOptions, args); 1413 if (cmdLine.hasOption(helpOpt.getOpt()) 1414 || cmdLine.hasOption(helpOpt.getLongOpt())) { 1415 // should print out the help information 1416 out.println(helpDescription + "\n"); 1417 if (printGenericCommandUsage) { 1418 ToolRunner.printGenericCommandUsage(out); 1419 } 1420 return true; 1421 } 1422 } catch (ParseException pe) { 1423 return false; 1424 } 1425 } 1426 return false; 1427 } 1428 1429 /** 1430 * Get DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION from configuration. 1431 * 1432 * @param conf Configuration 1433 * @return Value of DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION 1434 */ 1435 public static float getInvalidateWorkPctPerIteration(Configuration conf) { 1436 float blocksInvalidateWorkPct = conf.getFloat( 1437 DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, 1438 DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT); 1439 Preconditions.checkArgument( 1440 (blocksInvalidateWorkPct > 0 && blocksInvalidateWorkPct <= 1.0f), 1441 DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION + 1442 " = '" + blocksInvalidateWorkPct + "' is invalid. " + 1443 "It should be a positive, non-zero float value, not greater than 1.0f, " + 1444 "to indicate a percentage."); 1445 return blocksInvalidateWorkPct; 1446 } 1447 1448 /** 1449 * Get DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION from 1450 * configuration. 1451 * 1452 * @param conf Configuration 1453 * @return Value of DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION 1454 */ 1455 public static int getReplWorkMultiplier(Configuration conf) { 1456 int blocksReplWorkMultiplier = conf.getInt( 1457 DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION, 1458 DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT); 1459 Preconditions.checkArgument( 1460 (blocksReplWorkMultiplier > 0), 1461 DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION + 1462 " = '" + blocksReplWorkMultiplier + "' is invalid. " + 1463 "It should be a positive, non-zero integer value."); 1464 return blocksReplWorkMultiplier; 1465 } 1466 1467 /** 1468 * Get SPNEGO keytab Key from configuration 1469 * 1470 * @param conf 1471 * Configuration 1472 * @param defaultKey 1473 * @return DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY if the key is not empty 1474 * else return defaultKey 1475 */ 1476 public static String getSpnegoKeytabKey(Configuration conf, String defaultKey) { 1477 String value = 1478 conf.get(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY); 1479 return (value == null || value.isEmpty()) ? 1480 defaultKey : DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY; 1481 } 1482 1483 /** 1484 * Get http policy. Http Policy is chosen as follows: 1485 * <ol> 1486 * <li>If hadoop.ssl.enabled is set, http endpoints are not started. Only 1487 * https endpoints are started on configured https ports</li> 1488 * <li>This configuration is overridden by dfs.https.enable configuration, if 1489 * it is set to true. In that case, both http and https endpoints are stared.</li> 1490 * <li>All the above configurations are overridden by dfs.http.policy 1491 * configuration. With this configuration you can set http-only, https-only 1492 * and http-and-https endpoints.</li> 1493 * </ol> 1494 * See hdfs-default.xml documentation for more details on each of the above 1495 * configuration settings. 1496 */ 1497 public static HttpConfig.Policy getHttpPolicy(Configuration conf) { 1498 String policyStr = conf.get(DFSConfigKeys.DFS_HTTP_POLICY_KEY); 1499 if (policyStr == null) { 1500 boolean https = conf.getBoolean(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY, 1501 DFSConfigKeys.DFS_HTTPS_ENABLE_DEFAULT); 1502 1503 boolean hadoopSsl = conf.getBoolean( 1504 CommonConfigurationKeys.HADOOP_SSL_ENABLED_KEY, 1505 CommonConfigurationKeys.HADOOP_SSL_ENABLED_DEFAULT); 1506 1507 if (hadoopSsl) { 1508 LOG.warn(CommonConfigurationKeys.HADOOP_SSL_ENABLED_KEY 1509 + " is deprecated. Please use " + DFSConfigKeys.DFS_HTTP_POLICY_KEY 1510 + "."); 1511 } 1512 if (https) { 1513 LOG.warn(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY 1514 + " is deprecated. Please use " + DFSConfigKeys.DFS_HTTP_POLICY_KEY 1515 + "."); 1516 } 1517 1518 return (hadoopSsl || https) ? HttpConfig.Policy.HTTP_AND_HTTPS 1519 : HttpConfig.Policy.HTTP_ONLY; 1520 } 1521 1522 HttpConfig.Policy policy = HttpConfig.Policy.fromString(policyStr); 1523 if (policy == null) { 1524 throw new HadoopIllegalArgumentException("Unregonized value '" 1525 + policyStr + "' for " + DFSConfigKeys.DFS_HTTP_POLICY_KEY); 1526 } 1527 1528 conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, policy.name()); 1529 return policy; 1530 } 1531 1532 public static HttpServer2.Builder loadSslConfToHttpServerBuilder(HttpServer2.Builder builder, 1533 Configuration sslConf) { 1534 return builder 1535 .needsClientAuth( 1536 sslConf.getBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY, 1537 DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT)) 1538 .keyPassword(sslConf.get("ssl.server.keystore.keypassword")) 1539 .keyStore(sslConf.get("ssl.server.keystore.location"), 1540 sslConf.get("ssl.server.keystore.password"), 1541 sslConf.get("ssl.server.keystore.type", "jks")) 1542 .trustStore(sslConf.get("ssl.server.truststore.location"), 1543 sslConf.get("ssl.server.truststore.password"), 1544 sslConf.get("ssl.server.truststore.type", "jks")); 1545 } 1546 1547 /** 1548 * Load HTTPS-related configuration. 1549 */ 1550 public static Configuration loadSslConfiguration(Configuration conf) { 1551 Configuration sslConf = new Configuration(false); 1552 1553 sslConf.addResource(conf.get( 1554 DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY, 1555 DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_DEFAULT)); 1556 1557 boolean requireClientAuth = conf.getBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY, 1558 DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT); 1559 sslConf.setBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY, requireClientAuth); 1560 return sslConf; 1561 } 1562 1563 /** 1564 * Return a HttpServer.Builder that the journalnode / namenode / secondary 1565 * namenode can use to initialize their HTTP / HTTPS server. 1566 * 1567 */ 1568 public static HttpServer2.Builder httpServerTemplateForNNAndJN( 1569 Configuration conf, final InetSocketAddress httpAddr, 1570 final InetSocketAddress httpsAddr, String name, String spnegoUserNameKey, 1571 String spnegoKeytabFileKey) throws IOException { 1572 HttpConfig.Policy policy = getHttpPolicy(conf); 1573 1574 HttpServer2.Builder builder = new HttpServer2.Builder().setName(name) 1575 .setConf(conf).setACL(new AccessControlList(conf.get(DFS_ADMIN, " "))) 1576 .setSecurityEnabled(UserGroupInformation.isSecurityEnabled()) 1577 .setUsernameConfKey(spnegoUserNameKey) 1578 .setKeytabConfKey(getSpnegoKeytabKey(conf, spnegoKeytabFileKey)); 1579 1580 // initialize the webserver for uploading/downloading files. 1581 LOG.info("Starting web server as: " 1582 + SecurityUtil.getServerPrincipal(conf.get(spnegoUserNameKey), 1583 httpAddr.getHostName())); 1584 1585 if (policy.isHttpEnabled()) { 1586 if (httpAddr.getPort() == 0) { 1587 builder.setFindPort(true); 1588 } 1589 1590 URI uri = URI.create("http://" + NetUtils.getHostPortString(httpAddr)); 1591 builder.addEndpoint(uri); 1592 LOG.info("Starting Web-server for " + name + " at: " + uri); 1593 } 1594 1595 if (policy.isHttpsEnabled() && httpsAddr != null) { 1596 Configuration sslConf = loadSslConfiguration(conf); 1597 loadSslConfToHttpServerBuilder(builder, sslConf); 1598 1599 if (httpsAddr.getPort() == 0) { 1600 builder.setFindPort(true); 1601 } 1602 1603 URI uri = URI.create("https://" + NetUtils.getHostPortString(httpsAddr)); 1604 builder.addEndpoint(uri); 1605 LOG.info("Starting Web-server for " + name + " at: " + uri); 1606 } 1607 return builder; 1608 } 1609 1610 /** 1611 * Converts a Date into an ISO-8601 formatted datetime string. 1612 */ 1613 public static String dateToIso8601String(Date date) { 1614 SimpleDateFormat df = 1615 new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.ENGLISH); 1616 return df.format(date); 1617 } 1618 1619 /** 1620 * Converts a time duration in milliseconds into DDD:HH:MM:SS format. 1621 */ 1622 public static String durationToString(long durationMs) { 1623 boolean negative = false; 1624 if (durationMs < 0) { 1625 negative = true; 1626 durationMs = -durationMs; 1627 } 1628 // Chop off the milliseconds 1629 long durationSec = durationMs / 1000; 1630 final int secondsPerMinute = 60; 1631 final int secondsPerHour = 60*60; 1632 final int secondsPerDay = 60*60*24; 1633 final long days = durationSec / secondsPerDay; 1634 durationSec -= days * secondsPerDay; 1635 final long hours = durationSec / secondsPerHour; 1636 durationSec -= hours * secondsPerHour; 1637 final long minutes = durationSec / secondsPerMinute; 1638 durationSec -= minutes * secondsPerMinute; 1639 final long seconds = durationSec; 1640 final long milliseconds = durationMs % 1000; 1641 String format = "%03d:%02d:%02d:%02d.%03d"; 1642 if (negative) { 1643 format = "-" + format; 1644 } 1645 return String.format(format, days, hours, minutes, seconds, milliseconds); 1646 } 1647 1648 /** 1649 * Converts a relative time string into a duration in milliseconds. 1650 */ 1651 public static long parseRelativeTime(String relTime) throws IOException { 1652 if (relTime.length() < 2) { 1653 throw new IOException("Unable to parse relative time value of " + relTime 1654 + ": too short"); 1655 } 1656 String ttlString = relTime.substring(0, relTime.length()-1); 1657 long ttl; 1658 try { 1659 ttl = Long.parseLong(ttlString); 1660 } catch (NumberFormatException e) { 1661 throw new IOException("Unable to parse relative time value of " + relTime 1662 + ": " + ttlString + " is not a number"); 1663 } 1664 if (relTime.endsWith("s")) { 1665 // pass 1666 } else if (relTime.endsWith("m")) { 1667 ttl *= 60; 1668 } else if (relTime.endsWith("h")) { 1669 ttl *= 60*60; 1670 } else if (relTime.endsWith("d")) { 1671 ttl *= 60*60*24; 1672 } else { 1673 throw new IOException("Unable to parse relative time value of " + relTime 1674 + ": unknown time unit " + relTime.charAt(relTime.length() - 1)); 1675 } 1676 return ttl*1000; 1677 } 1678}