001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hdfs;
020
021import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ADMIN;
022import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT;
023import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_HTTPS_NEED_AUTH_KEY;
024import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
025import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODE_ID_KEY;
026import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY;
027import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT;
028import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY;
029import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_DEFAULT;
030import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY;
031import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY;
032import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY;
033import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY;
034import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
035import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
036
037import java.io.IOException;
038import java.io.PrintStream;
039import java.io.UnsupportedEncodingException;
040import java.net.InetSocketAddress;
041import java.net.URI;
042import java.net.URISyntaxException;
043import java.security.SecureRandom;
044import java.text.SimpleDateFormat;
045import java.util.ArrayList;
046import java.util.Collection;
047import java.util.Collections;
048import java.util.Comparator;
049import java.util.Date;
050import java.util.HashSet;
051import java.util.List;
052import java.util.Locale;
053import java.util.Map;
054import java.util.Random;
055import java.util.Set;
056import java.util.concurrent.TimeUnit;
057
058import javax.net.SocketFactory;
059
060import org.apache.commons.cli.CommandLine;
061import org.apache.commons.cli.CommandLineParser;
062import org.apache.commons.cli.Option;
063import org.apache.commons.cli.Options;
064import org.apache.commons.cli.ParseException;
065import org.apache.commons.cli.PosixParser;
066import org.apache.commons.logging.Log;
067import org.apache.commons.logging.LogFactory;
068import org.apache.hadoop.HadoopIllegalArgumentException;
069import org.apache.hadoop.classification.InterfaceAudience;
070import org.apache.hadoop.conf.Configuration;
071import org.apache.hadoop.fs.BlockLocation;
072import org.apache.hadoop.fs.CommonConfigurationKeys;
073import org.apache.hadoop.fs.FileSystem;
074import org.apache.hadoop.fs.Path;
075import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
076import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
077import org.apache.hadoop.hdfs.protocol.DatanodeID;
078import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
079import org.apache.hadoop.hdfs.protocol.HdfsConstants;
080import org.apache.hadoop.hdfs.protocol.LocatedBlock;
081import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
082import org.apache.hadoop.hdfs.protocolPB.ClientDatanodeProtocolTranslatorPB;
083import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
084import org.apache.hadoop.hdfs.server.namenode.NameNode;
085import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem;
086import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
087import org.apache.hadoop.http.HttpConfig;
088import org.apache.hadoop.http.HttpServer2;
089import org.apache.hadoop.ipc.ProtobufRpcEngine;
090import org.apache.hadoop.ipc.RPC;
091import org.apache.hadoop.net.NetUtils;
092import org.apache.hadoop.net.NodeBase;
093import org.apache.hadoop.security.SecurityUtil;
094import org.apache.hadoop.security.UserGroupInformation;
095import org.apache.hadoop.security.authorize.AccessControlList;
096import org.apache.hadoop.util.StringUtils;
097import org.apache.hadoop.util.ToolRunner;
098
099import com.google.common.annotations.VisibleForTesting;
100import com.google.common.base.Charsets;
101import com.google.common.base.Joiner;
102import com.google.common.base.Preconditions;
103import com.google.common.collect.Lists;
104import com.google.common.collect.Maps;
105import com.google.common.primitives.SignedBytes;
106import com.google.protobuf.BlockingService;
107
108@InterfaceAudience.Private
109public class DFSUtil {
110  public static final Log LOG = LogFactory.getLog(DFSUtil.class.getName());
111  
112  public static final byte[] EMPTY_BYTES = {};
113
114  /** Compare two byte arrays by lexicographical order. */
115  public static int compareBytes(byte[] left, byte[] right) {
116    if (left == null) {
117      left = EMPTY_BYTES;
118    }
119    if (right == null) {
120      right = EMPTY_BYTES;
121    }
122    return SignedBytes.lexicographicalComparator().compare(left, right);
123  }
124
125  private DFSUtil() { /* Hidden constructor */ }
126  private static final ThreadLocal<Random> RANDOM = new ThreadLocal<Random>() {
127    @Override
128    protected Random initialValue() {
129      return new Random();
130    }
131  };
132  
133  private static final ThreadLocal<SecureRandom> SECURE_RANDOM = new ThreadLocal<SecureRandom>() {
134    @Override
135    protected SecureRandom initialValue() {
136      return new SecureRandom();
137    }
138  };
139
140  /** @return a pseudo random number generator. */
141  public static Random getRandom() {
142    return RANDOM.get();
143  }
144  
145  /** @return a pseudo secure random number generator. */
146  public static SecureRandom getSecureRandom() {
147    return SECURE_RANDOM.get();
148  }
149
150  /** Shuffle the elements in the given array. */
151  public static <T> T[] shuffle(final T[] array) {
152    if (array != null && array.length > 0) {
153      final Random random = getRandom();
154      for (int n = array.length; n > 1; ) {
155        final int randomIndex = random.nextInt(n);
156        n--;
157        if (n != randomIndex) {
158          final T tmp = array[randomIndex];
159          array[randomIndex] = array[n];
160          array[n] = tmp;
161        }
162      }
163    }
164    return array;
165  }
166
167  /**
168   * Compartor for sorting DataNodeInfo[] based on decommissioned states.
169   * Decommissioned nodes are moved to the end of the array on sorting with
170   * this compartor.
171   */
172  public static final Comparator<DatanodeInfo> DECOM_COMPARATOR = 
173    new Comparator<DatanodeInfo>() {
174      @Override
175      public int compare(DatanodeInfo a, DatanodeInfo b) {
176        return a.isDecommissioned() == b.isDecommissioned() ? 0 : 
177          a.isDecommissioned() ? 1 : -1;
178      }
179    };
180    
181      
182  /**
183   * Comparator for sorting DataNodeInfo[] based on decommissioned/stale states.
184   * Decommissioned/stale nodes are moved to the end of the array on sorting
185   * with this comparator.
186   */ 
187  @InterfaceAudience.Private 
188  public static class DecomStaleComparator implements Comparator<DatanodeInfo> {
189    private long staleInterval;
190
191    /**
192     * Constructor of DecomStaleComparator
193     * 
194     * @param interval
195     *          The time interval for marking datanodes as stale is passed from
196     *          outside, since the interval may be changed dynamically
197     */
198    public DecomStaleComparator(long interval) {
199      this.staleInterval = interval;
200    }
201
202    @Override
203    public int compare(DatanodeInfo a, DatanodeInfo b) {
204      // Decommissioned nodes will still be moved to the end of the list
205      if (a.isDecommissioned()) {
206        return b.isDecommissioned() ? 0 : 1;
207      } else if (b.isDecommissioned()) {
208        return -1;
209      }
210      // Stale nodes will be moved behind the normal nodes
211      boolean aStale = a.isStale(staleInterval);
212      boolean bStale = b.isStale(staleInterval);
213      return aStale == bStale ? 0 : (aStale ? 1 : -1);
214    }
215  }    
216    
217  /**
218   * Address matcher for matching an address to local address
219   */
220  static final AddressMatcher LOCAL_ADDRESS_MATCHER = new AddressMatcher() {
221    @Override
222    public boolean match(InetSocketAddress s) {
223      return NetUtils.isLocalAddress(s.getAddress());
224    };
225  };
226  
227  /**
228   * Whether the pathname is valid.  Currently prohibits relative paths, 
229   * names which contain a ":" or "//", or other non-canonical paths.
230   */
231  public static boolean isValidName(String src) {
232    // Path must be absolute.
233    if (!src.startsWith(Path.SEPARATOR)) {
234      return false;
235    }
236      
237    // Check for ".." "." ":" "/"
238    String[] components = StringUtils.split(src, '/');
239    for (int i = 0; i < components.length; i++) {
240      String element = components[i];
241      if (element.equals(".")  ||
242          (element.indexOf(":") >= 0)  ||
243          (element.indexOf("/") >= 0)) {
244        return false;
245      }
246      // ".." is allowed in path starting with /.reserved/.inodes
247      if (element.equals("..")) {
248        if (components.length > 4
249            && components[1].equals(FSDirectory.DOT_RESERVED_STRING)
250            && components[2].equals(FSDirectory.DOT_INODES_STRING)) {
251          continue;
252        }
253        return false;
254      }
255      // The string may start or end with a /, but not have
256      // "//" in the middle.
257      if (element.isEmpty() && i != components.length - 1 &&
258          i != 0) {
259        return false;
260      }
261    }
262    return true;
263  }
264
265  /**
266   * Converts a byte array to a string using UTF8 encoding.
267   */
268  public static String bytes2String(byte[] bytes) {
269    return bytes2String(bytes, 0, bytes.length);
270  }
271  
272  /**
273   * Decode a specific range of bytes of the given byte array to a string
274   * using UTF8.
275   * 
276   * @param bytes The bytes to be decoded into characters
277   * @param offset The index of the first byte to decode
278   * @param length The number of bytes to decode
279   * @return The decoded string
280   */
281  public static String bytes2String(byte[] bytes, int offset, int length) {
282    try {
283      return new String(bytes, offset, length, "UTF8");
284    } catch(UnsupportedEncodingException e) {
285      assert false : "UTF8 encoding is not supported ";
286    }
287    return null;
288  }
289
290  /**
291   * Converts a string to a byte array using UTF8 encoding.
292   */
293  public static byte[] string2Bytes(String str) {
294    return str.getBytes(Charsets.UTF_8);
295  }
296
297  /**
298   * Given a list of path components returns a path as a UTF8 String
299   */
300  public static String byteArray2PathString(byte[][] pathComponents) {
301    if (pathComponents.length == 0) {
302      return "";
303    } else if (pathComponents.length == 1
304        && (pathComponents[0] == null || pathComponents[0].length == 0)) {
305      return Path.SEPARATOR;
306    }
307    StringBuilder result = new StringBuilder();
308    for (int i = 0; i < pathComponents.length; i++) {
309      result.append(new String(pathComponents[i], Charsets.UTF_8));
310      if (i < pathComponents.length - 1) {
311        result.append(Path.SEPARATOR_CHAR);
312      }
313    }
314    return result.toString();
315  }
316  
317  /**
318   * Given a list of path components returns a byte array
319   */
320  public static byte[] byteArray2bytes(byte[][] pathComponents) {
321    if (pathComponents.length == 0) {
322      return EMPTY_BYTES;
323    } else if (pathComponents.length == 1
324        && (pathComponents[0] == null || pathComponents[0].length == 0)) {
325      return new byte[]{(byte) Path.SEPARATOR_CHAR};
326    }
327    int length = 0;
328    for (int i = 0; i < pathComponents.length; i++) {
329      length += pathComponents[i].length;
330      if (i < pathComponents.length - 1) {
331        length++; // for SEPARATOR
332      }
333    }
334    byte[] path = new byte[length];
335    int index = 0;
336    for (int i = 0; i < pathComponents.length; i++) {
337      System.arraycopy(pathComponents[i], 0, path, index,
338          pathComponents[i].length);
339      index += pathComponents[i].length;
340      if (i < pathComponents.length - 1) {
341        path[index] = (byte) Path.SEPARATOR_CHAR;
342        index++;
343      }
344    }
345    return path;
346  }
347
348  /** Convert an object representing a path to a string. */
349  public static String path2String(final Object path) {
350    return path == null? null
351        : path instanceof String? (String)path
352        : path instanceof byte[][]? byteArray2PathString((byte[][])path)
353        : path.toString();
354  }
355
356  /**
357   * Splits the array of bytes into array of arrays of bytes
358   * on byte separator
359   * @param bytes the array of bytes to split
360   * @param separator the delimiting byte
361   */
362  public static byte[][] bytes2byteArray(byte[] bytes, byte separator) {
363    return bytes2byteArray(bytes, bytes.length, separator);
364  }
365
366  /**
367   * Splits first len bytes in bytes to array of arrays of bytes
368   * on byte separator
369   * @param bytes the byte array to split
370   * @param len the number of bytes to split
371   * @param separator the delimiting byte
372   */
373  public static byte[][] bytes2byteArray(byte[] bytes,
374                                         int len,
375                                         byte separator) {
376    assert len <= bytes.length;
377    int splits = 0;
378    if (len == 0) {
379      return new byte[][]{null};
380    }
381    // Count the splits. Omit multiple separators and the last one
382    for (int i = 0; i < len; i++) {
383      if (bytes[i] == separator) {
384        splits++;
385      }
386    }
387    int last = len - 1;
388    while (last > -1 && bytes[last--] == separator) {
389      splits--;
390    }
391    if (splits == 0 && bytes[0] == separator) {
392      return new byte[][]{null};
393    }
394    splits++;
395    byte[][] result = new byte[splits][];
396    int startIndex = 0;
397    int nextIndex = 0;
398    int index = 0;
399    // Build the splits
400    while (index < splits) {
401      while (nextIndex < len && bytes[nextIndex] != separator) {
402        nextIndex++;
403      }
404      result[index] = new byte[nextIndex - startIndex];
405      System.arraycopy(bytes, startIndex, result[index], 0, nextIndex
406              - startIndex);
407      index++;
408      startIndex = nextIndex + 1;
409      nextIndex = startIndex;
410    }
411    return result;
412  }
413  
414  /**
415   * Convert a LocatedBlocks to BlockLocations[]
416   * @param blocks a LocatedBlocks
417   * @return an array of BlockLocations
418   */
419  public static BlockLocation[] locatedBlocks2Locations(LocatedBlocks blocks) {
420    if (blocks == null) {
421      return new BlockLocation[0];
422    }
423    return locatedBlocks2Locations(blocks.getLocatedBlocks());
424  }
425  
426  /**
427   * Convert a List<LocatedBlock> to BlockLocation[]
428   * @param blocks A List<LocatedBlock> to be converted
429   * @return converted array of BlockLocation
430   */
431  public static BlockLocation[] locatedBlocks2Locations(List<LocatedBlock> blocks) {
432    if (blocks == null) {
433      return new BlockLocation[0];
434    }
435    int nrBlocks = blocks.size();
436    BlockLocation[] blkLocations = new BlockLocation[nrBlocks];
437    if (nrBlocks == 0) {
438      return blkLocations;
439    }
440    int idx = 0;
441    for (LocatedBlock blk : blocks) {
442      assert idx < nrBlocks : "Incorrect index";
443      DatanodeInfo[] locations = blk.getLocations();
444      String[] hosts = new String[locations.length];
445      String[] xferAddrs = new String[locations.length];
446      String[] racks = new String[locations.length];
447      for (int hCnt = 0; hCnt < locations.length; hCnt++) {
448        hosts[hCnt] = locations[hCnt].getHostName();
449        xferAddrs[hCnt] = locations[hCnt].getXferAddr();
450        NodeBase node = new NodeBase(xferAddrs[hCnt], 
451                                     locations[hCnt].getNetworkLocation());
452        racks[hCnt] = node.toString();
453      }
454      DatanodeInfo[] cachedLocations = blk.getCachedLocations();
455      String[] cachedHosts = new String[cachedLocations.length];
456      for (int i=0; i<cachedLocations.length; i++) {
457        cachedHosts[i] = cachedLocations[i].getHostName();
458      }
459      blkLocations[idx] = new BlockLocation(xferAddrs, hosts, cachedHosts,
460                                            racks,
461                                            blk.getStartOffset(),
462                                            blk.getBlockSize(),
463                                            blk.isCorrupt());
464      idx++;
465    }
466    return blkLocations;
467  }
468
469  /**
470   * Returns collection of nameservice Ids from the configuration.
471   * @param conf configuration
472   * @return collection of nameservice Ids, or null if not specified
473   */
474  public static Collection<String> getNameServiceIds(Configuration conf) {
475    return conf.getTrimmedStringCollection(DFS_NAMESERVICES);
476  }
477
478  /**
479   * @return <code>coll</code> if it is non-null and non-empty. Otherwise,
480   * returns a list with a single null value.
481   */
482  private static Collection<String> emptyAsSingletonNull(Collection<String> coll) {
483    if (coll == null || coll.isEmpty()) {
484      return Collections.singletonList(null);
485    } else {
486      return coll;
487    }
488  }
489  
490  /**
491   * Namenode HighAvailability related configuration.
492   * Returns collection of namenode Ids from the configuration. One logical id
493   * for each namenode in the in the HA setup.
494   * 
495   * @param conf configuration
496   * @param nsId the nameservice ID to look at, or null for non-federated 
497   * @return collection of namenode Ids
498   */
499  public static Collection<String> getNameNodeIds(Configuration conf, String nsId) {
500    String key = addSuffix(DFS_HA_NAMENODES_KEY_PREFIX, nsId);
501    return conf.getTrimmedStringCollection(key);
502  }
503  
504  /**
505   * Given a list of keys in the order of preference, returns a value
506   * for the key in the given order from the configuration.
507   * @param defaultValue default value to return, when key was not found
508   * @param keySuffix suffix to add to the key, if it is not null
509   * @param conf Configuration
510   * @param keys list of keys in the order of preference
511   * @return value of the key or default if a key was not found in configuration
512   */
513  private static String getConfValue(String defaultValue, String keySuffix,
514      Configuration conf, String... keys) {
515    String value = null;
516    for (String key : keys) {
517      key = addSuffix(key, keySuffix);
518      value = conf.get(key);
519      if (value != null) {
520        break;
521      }
522    }
523    if (value == null) {
524      value = defaultValue;
525    }
526    return value;
527  }
528  
529  /** Add non empty and non null suffix to a key */
530  private static String addSuffix(String key, String suffix) {
531    if (suffix == null || suffix.isEmpty()) {
532      return key;
533    }
534    assert !suffix.startsWith(".") :
535      "suffix '" + suffix + "' should not already have '.' prepended.";
536    return key + "." + suffix;
537  }
538  
539  /** Concatenate list of suffix strings '.' separated */
540  private static String concatSuffixes(String... suffixes) {
541    if (suffixes == null) {
542      return null;
543    }
544    return Joiner.on(".").skipNulls().join(suffixes);
545  }
546  
547  /**
548   * Return configuration key of format key.suffix1.suffix2...suffixN
549   */
550  public static String addKeySuffixes(String key, String... suffixes) {
551    String keySuffix = concatSuffixes(suffixes);
552    return addSuffix(key, keySuffix);
553  }
554  
555  /**
556   * Returns the configured address for all NameNodes in the cluster.
557   * @param conf configuration
558   * @param defaultAddress default address to return in case key is not found.
559   * @param keys Set of keys to look for in the order of preference
560   * @return a map(nameserviceId to map(namenodeId to InetSocketAddress))
561   */
562  private static Map<String, Map<String, InetSocketAddress>>
563    getAddresses(Configuration conf,
564      String defaultAddress, String... keys) {
565    Collection<String> nameserviceIds = getNameServiceIds(conf);
566    
567    // Look for configurations of the form <key>[.<nameserviceId>][.<namenodeId>]
568    // across all of the configured nameservices and namenodes.
569    Map<String, Map<String, InetSocketAddress>> ret = Maps.newLinkedHashMap();
570    for (String nsId : emptyAsSingletonNull(nameserviceIds)) {
571      Map<String, InetSocketAddress> isas =
572        getAddressesForNameserviceId(conf, nsId, defaultAddress, keys);
573      if (!isas.isEmpty()) {
574        ret.put(nsId, isas);
575      }
576    }
577    return ret;
578  }
579
580  private static Map<String, InetSocketAddress> getAddressesForNameserviceId(
581      Configuration conf, String nsId, String defaultValue,
582      String[] keys) {
583    Collection<String> nnIds = getNameNodeIds(conf, nsId);
584    Map<String, InetSocketAddress> ret = Maps.newHashMap();
585    for (String nnId : emptyAsSingletonNull(nnIds)) {
586      String suffix = concatSuffixes(nsId, nnId);
587      String address = getConfValue(defaultValue, suffix, conf, keys);
588      if (address != null) {
589        InetSocketAddress isa = NetUtils.createSocketAddr(address);
590        if (isa.isUnresolved()) {
591          LOG.warn("Namenode for " + nsId +
592                   " remains unresolved for ID " + nnId +
593                   ".  Check your hdfs-site.xml file to " +
594                   "ensure namenodes are configured properly.");
595        }
596        ret.put(nnId, isa);
597      }
598    }
599    return ret;
600  }
601
602  /**
603   * @return a collection of all configured NN Kerberos principals.
604   */
605  public static Set<String> getAllNnPrincipals(Configuration conf) throws IOException {
606    Set<String> principals = new HashSet<String>();
607    for (String nsId : DFSUtil.getNameServiceIds(conf)) {
608      if (HAUtil.isHAEnabled(conf, nsId)) {
609        for (String nnId : DFSUtil.getNameNodeIds(conf, nsId)) {
610          Configuration confForNn = new Configuration(conf);
611          NameNode.initializeGenericKeys(confForNn, nsId, nnId);
612          String principal = SecurityUtil.getServerPrincipal(confForNn
613              .get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY),
614              NameNode.getAddress(confForNn).getHostName());
615          principals.add(principal);
616        }
617      } else {
618        Configuration confForNn = new Configuration(conf);
619        NameNode.initializeGenericKeys(confForNn, nsId, null);
620        String principal = SecurityUtil.getServerPrincipal(confForNn
621            .get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY),
622            NameNode.getAddress(confForNn).getHostName());
623        principals.add(principal);
624      }
625    }
626
627    return principals;
628  }
629
630  /**
631   * Returns list of InetSocketAddress corresponding to HA NN RPC addresses from
632   * the configuration.
633   * 
634   * @param conf configuration
635   * @return list of InetSocketAddresses
636   */
637  public static Map<String, Map<String, InetSocketAddress>> getHaNnRpcAddresses(
638      Configuration conf) {
639    return getAddresses(conf, null, DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
640  }
641
642  /**
643   * Returns list of InetSocketAddress corresponding to HA NN HTTP addresses from
644   * the configuration.
645   *
646   * @return list of InetSocketAddresses
647   */
648  public static Map<String, Map<String, InetSocketAddress>> getHaNnWebHdfsAddresses(
649      Configuration conf, String scheme) {
650    if (WebHdfsFileSystem.SCHEME.equals(scheme)) {
651      return getAddresses(conf, null,
652          DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY);
653    } else if (SWebHdfsFileSystem.SCHEME.equals(scheme)) {
654      return getAddresses(conf, null,
655          DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY);
656    } else {
657      throw new IllegalArgumentException("Unsupported scheme: " + scheme);
658    }
659  }
660
661  /**
662   * Resolve an HDFS URL into real INetSocketAddress. It works like a DNS resolver
663   * when the URL points to an non-HA cluster. When the URL points to an HA
664   * cluster, the resolver further resolves the logical name (i.e., the authority
665   * in the URL) into real namenode addresses.
666   */
667  public static InetSocketAddress[] resolveWebHdfsUri(URI uri, Configuration conf)
668      throws IOException {
669    int defaultPort;
670    String scheme = uri.getScheme();
671    if (WebHdfsFileSystem.SCHEME.equals(scheme)) {
672      defaultPort = DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT;
673    } else if (SWebHdfsFileSystem.SCHEME.equals(scheme)) {
674      defaultPort = DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT;
675    } else {
676      throw new IllegalArgumentException("Unsupported scheme: " + scheme);
677    }
678
679    ArrayList<InetSocketAddress> ret = new ArrayList<InetSocketAddress>();
680
681    if (!HAUtil.isLogicalUri(conf, uri)) {
682      InetSocketAddress addr = NetUtils.createSocketAddr(uri.getAuthority(),
683          defaultPort);
684      ret.add(addr);
685
686    } else {
687      Map<String, Map<String, InetSocketAddress>> addresses = DFSUtil
688          .getHaNnWebHdfsAddresses(conf, scheme);
689
690      for (Map<String, InetSocketAddress> addrs : addresses.values()) {
691        for (InetSocketAddress addr : addrs.values()) {
692          ret.add(addr);
693        }
694      }
695    }
696
697    InetSocketAddress[] r = new InetSocketAddress[ret.size()];
698    return ret.toArray(r);
699  }
700  
701  /**
702   * Returns list of InetSocketAddress corresponding to  backup node rpc 
703   * addresses from the configuration.
704   * 
705   * @param conf configuration
706   * @return list of InetSocketAddresses
707   * @throws IOException on error
708   */
709  public static Map<String, Map<String, InetSocketAddress>> getBackupNodeAddresses(
710      Configuration conf) throws IOException {
711    Map<String, Map<String, InetSocketAddress>> addressList = getAddresses(conf,
712        null, DFS_NAMENODE_BACKUP_ADDRESS_KEY);
713    if (addressList.isEmpty()) {
714      throw new IOException("Incorrect configuration: backup node address "
715          + DFS_NAMENODE_BACKUP_ADDRESS_KEY + " is not configured.");
716    }
717    return addressList;
718  }
719
720  /**
721   * Returns list of InetSocketAddresses of corresponding to secondary namenode
722   * http addresses from the configuration.
723   * 
724   * @param conf configuration
725   * @return list of InetSocketAddresses
726   * @throws IOException on error
727   */
728  public static Map<String, Map<String, InetSocketAddress>> getSecondaryNameNodeAddresses(
729      Configuration conf) throws IOException {
730    Map<String, Map<String, InetSocketAddress>> addressList = getAddresses(conf, null,
731        DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY);
732    if (addressList.isEmpty()) {
733      throw new IOException("Incorrect configuration: secondary namenode address "
734          + DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY + " is not configured.");
735    }
736    return addressList;
737  }
738
739  /**
740   * Returns list of InetSocketAddresses corresponding to namenodes from the
741   * configuration. Note this is to be used by datanodes to get the list of
742   * namenode addresses to talk to.
743   * 
744   * Returns namenode address specifically configured for datanodes (using
745   * service ports), if found. If not, regular RPC address configured for other
746   * clients is returned.
747   * 
748   * @param conf configuration
749   * @return list of InetSocketAddress
750   * @throws IOException on error
751   */
752  public static Map<String, Map<String, InetSocketAddress>> getNNServiceRpcAddresses(
753      Configuration conf) throws IOException {
754    // Use default address as fall back
755    String defaultAddress;
756    try {
757      defaultAddress = NetUtils.getHostPortString(NameNode.getAddress(conf));
758    } catch (IllegalArgumentException e) {
759      defaultAddress = null;
760    }
761    
762    Map<String, Map<String, InetSocketAddress>> addressList =
763      getAddresses(conf, defaultAddress,
764        DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, DFS_NAMENODE_RPC_ADDRESS_KEY);
765    if (addressList.isEmpty()) {
766      throw new IOException("Incorrect configuration: namenode address "
767          + DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY + " or "  
768          + DFS_NAMENODE_RPC_ADDRESS_KEY
769          + " is not configured.");
770    }
771    return addressList;
772  }
773  
774  /**
775   * Flatten the given map, as returned by other functions in this class,
776   * into a flat list of {@link ConfiguredNNAddress} instances.
777   */
778  public static List<ConfiguredNNAddress> flattenAddressMap(
779      Map<String, Map<String, InetSocketAddress>> map) {
780    List<ConfiguredNNAddress> ret = Lists.newArrayList();
781    
782    for (Map.Entry<String, Map<String, InetSocketAddress>> entry :
783      map.entrySet()) {
784      String nsId = entry.getKey();
785      Map<String, InetSocketAddress> nnMap = entry.getValue();
786      for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) {
787        String nnId = e2.getKey();
788        InetSocketAddress addr = e2.getValue();
789        
790        ret.add(new ConfiguredNNAddress(nsId, nnId, addr));
791      }
792    }
793    return ret;
794  }
795
796  /**
797   * Format the given map, as returned by other functions in this class,
798   * into a string suitable for debugging display. The format of this string
799   * should not be considered an interface, and is liable to change.
800   */
801  public static String addressMapToString(
802      Map<String, Map<String, InetSocketAddress>> map) {
803    StringBuilder b = new StringBuilder();
804    for (Map.Entry<String, Map<String, InetSocketAddress>> entry :
805         map.entrySet()) {
806      String nsId = entry.getKey();
807      Map<String, InetSocketAddress> nnMap = entry.getValue();
808      b.append("Nameservice <").append(nsId).append(">:").append("\n");
809      for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) {
810        b.append("  NN ID ").append(e2.getKey())
811          .append(" => ").append(e2.getValue()).append("\n");
812      }
813    }
814    return b.toString();
815  }
816  
817  public static String nnAddressesAsString(Configuration conf) {
818    Map<String, Map<String, InetSocketAddress>> addresses =
819      getHaNnRpcAddresses(conf);
820    return addressMapToString(addresses);
821  }
822
823  /**
824   * Represent one of the NameNodes configured in the cluster.
825   */
826  public static class ConfiguredNNAddress {
827    private final String nameserviceId;
828    private final String namenodeId;
829    private final InetSocketAddress addr;
830
831    private ConfiguredNNAddress(String nameserviceId, String namenodeId,
832        InetSocketAddress addr) {
833      this.nameserviceId = nameserviceId;
834      this.namenodeId = namenodeId;
835      this.addr = addr;
836    }
837
838    public String getNameserviceId() {
839      return nameserviceId;
840    }
841
842    public String getNamenodeId() {
843      return namenodeId;
844    }
845
846    public InetSocketAddress getAddress() {
847      return addr;
848    }
849    
850    @Override
851    public String toString() {
852      return "ConfiguredNNAddress[nsId=" + nameserviceId + ";" +
853        "nnId=" + namenodeId + ";addr=" + addr + "]";
854    }
855  }
856  
857  /**
858   * Get a URI for each configured nameservice. If a nameservice is
859   * HA-enabled, then the logical URI of the nameservice is returned. If the
860   * nameservice is not HA-enabled, then a URI corresponding to an RPC address
861   * of the single NN for that nameservice is returned, preferring the service
862   * RPC address over the client RPC address.
863   * 
864   * @param conf configuration
865   * @return a collection of all configured NN URIs, preferring service
866   *         addresses
867   */
868  public static Collection<URI> getNsServiceRpcUris(Configuration conf) {
869    return getNameServiceUris(conf,
870        DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
871        DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
872  }
873
874  /**
875   * Get a URI for each configured nameservice. If a nameservice is
876   * HA-enabled, then the logical URI of the nameservice is returned. If the
877   * nameservice is not HA-enabled, then a URI corresponding to the address of
878   * the single NN for that nameservice is returned.
879   * 
880   * @param conf configuration
881   * @param keys configuration keys to try in order to get the URI for non-HA
882   *        nameservices
883   * @return a collection of all configured NN URIs
884   */
885  public static Collection<URI> getNameServiceUris(Configuration conf,
886      String... keys) {
887    Set<URI> ret = new HashSet<URI>();
888    
889    // We're passed multiple possible configuration keys for any given NN or HA
890    // nameservice, and search the config in order of these keys. In order to
891    // make sure that a later config lookup (e.g. fs.defaultFS) doesn't add a
892    // URI for a config key for which we've already found a preferred entry, we
893    // keep track of non-preferred keys here.
894    Set<URI> nonPreferredUris = new HashSet<URI>();
895    
896    for (String nsId : getNameServiceIds(conf)) {
897      if (HAUtil.isHAEnabled(conf, nsId)) {
898        // Add the logical URI of the nameservice.
899        try {
900          ret.add(new URI(HdfsConstants.HDFS_URI_SCHEME + "://" + nsId));
901        } catch (URISyntaxException ue) {
902          throw new IllegalArgumentException(ue);
903        }
904      } else {
905        // Add the URI corresponding to the address of the NN.
906        boolean uriFound = false;
907        for (String key : keys) {
908          String addr = conf.get(concatSuffixes(key, nsId));
909          if (addr != null) {
910            URI uri = createUri(HdfsConstants.HDFS_URI_SCHEME,
911                NetUtils.createSocketAddr(addr));
912            if (!uriFound) {
913              uriFound = true;
914              ret.add(uri);
915            } else {
916              nonPreferredUris.add(uri);
917            }
918          }
919        }
920      }
921    }
922    
923    // Add the generic configuration keys.
924    boolean uriFound = false;
925    for (String key : keys) {
926      String addr = conf.get(key);
927      if (addr != null) {
928        URI uri = createUri("hdfs", NetUtils.createSocketAddr(addr));
929        if (!uriFound) {
930          uriFound = true;
931          ret.add(uri);
932        } else {
933          nonPreferredUris.add(uri);
934        }
935      }
936    }
937    
938    // Add the default URI if it is an HDFS URI.
939    URI defaultUri = FileSystem.getDefaultUri(conf);
940    // checks if defaultUri is ip:port format
941    // and convert it to hostname:port format
942    if (defaultUri != null && (defaultUri.getPort() != -1)) {
943      defaultUri = createUri(defaultUri.getScheme(),
944          NetUtils.createSocketAddr(defaultUri.getHost(), 
945              defaultUri.getPort()));
946    }
947    if (defaultUri != null &&
948        HdfsConstants.HDFS_URI_SCHEME.equals(defaultUri.getScheme()) &&
949        !nonPreferredUris.contains(defaultUri)) {
950      ret.add(defaultUri);
951    }
952    
953    return ret;
954  }
955
956  /**
957   * Given the InetSocketAddress this method returns the nameservice Id
958   * corresponding to the key with matching address, by doing a reverse 
959   * lookup on the list of nameservices until it finds a match.
960   * 
961   * Since the process of resolving URIs to Addresses is slightly expensive,
962   * this utility method should not be used in performance-critical routines.
963   * 
964   * @param conf - configuration
965   * @param address - InetSocketAddress for configured communication with NN.
966   *     Configured addresses are typically given as URIs, but we may have to
967   *     compare against a URI typed in by a human, or the server name may be
968   *     aliased, so we compare unambiguous InetSocketAddresses instead of just
969   *     comparing URI substrings.
970   * @param keys - list of configured communication parameters that should
971   *     be checked for matches.  For example, to compare against RPC addresses,
972   *     provide the list DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
973   *     DFS_NAMENODE_RPC_ADDRESS_KEY.  Use the generic parameter keys,
974   *     not the NameServiceId-suffixed keys.
975   * @return nameserviceId, or null if no match found
976   */
977  public static String getNameServiceIdFromAddress(final Configuration conf, 
978      final InetSocketAddress address, String... keys) {
979    // Configuration with a single namenode and no nameserviceId
980    String[] ids = getSuffixIDs(conf, address, keys);
981    return (ids != null) ? ids[0] : null;
982  }
983  
984  /**
985   * return server http or https address from the configuration for a
986   * given namenode rpc address.
987   * @param conf
988   * @param namenodeAddr - namenode RPC address
989   * @param scheme - the scheme (http / https)
990   * @return server http or https address
991   * @throws IOException 
992   */
993  public static URI getInfoServer(InetSocketAddress namenodeAddr,
994      Configuration conf, String scheme) throws IOException {
995    String[] suffixes = null;
996    if (namenodeAddr != null) {
997      // if non-default namenode, try reverse look up 
998      // the nameServiceID if it is available
999      suffixes = getSuffixIDs(conf, namenodeAddr,
1000          DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
1001          DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
1002    }
1003
1004    String authority;
1005    if ("http".equals(scheme)) {
1006      authority = getSuffixedConf(conf, DFS_NAMENODE_HTTP_ADDRESS_KEY,
1007          DFS_NAMENODE_HTTP_ADDRESS_DEFAULT, suffixes);
1008    } else if ("https".equals(scheme)) {
1009      authority = getSuffixedConf(conf, DFS_NAMENODE_HTTPS_ADDRESS_KEY,
1010          DFS_NAMENODE_HTTPS_ADDRESS_DEFAULT, suffixes);
1011    } else {
1012      throw new IllegalArgumentException("Invalid scheme:" + scheme);
1013    }
1014
1015    if (namenodeAddr != null) {
1016      authority = substituteForWildcardAddress(authority,
1017          namenodeAddr.getHostName());
1018    }
1019    return URI.create(scheme + "://" + authority);
1020  }
1021
1022  /**
1023   * Lookup the HTTP / HTTPS address of the namenode, and replace its hostname
1024   * with defaultHost when it found out that the address is a wildcard / local
1025   * address.
1026   *
1027   * @param defaultHost
1028   *          The default host name of the namenode.
1029   * @param conf
1030   *          The configuration
1031   * @param scheme
1032   *          HTTP or HTTPS
1033   * @throws IOException
1034   */
1035  public static URI getInfoServerWithDefaultHost(String defaultHost,
1036      Configuration conf, final String scheme) throws IOException {
1037    URI configuredAddr = getInfoServer(null, conf, scheme);
1038    String authority = substituteForWildcardAddress(
1039        configuredAddr.getAuthority(), defaultHost);
1040    return URI.create(scheme + "://" + authority);
1041  }
1042
1043  /**
1044   * Determine whether HTTP or HTTPS should be used to connect to the remote
1045   * server. Currently the client only connects to the server via HTTPS if the
1046   * policy is set to HTTPS_ONLY.
1047   *
1048   * @return the scheme (HTTP / HTTPS)
1049   */
1050  public static String getHttpClientScheme(Configuration conf) {
1051    HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf);
1052    return policy == HttpConfig.Policy.HTTPS_ONLY ? "https" : "http";
1053  }
1054
1055  /**
1056   * Substitute a default host in the case that an address has been configured
1057   * with a wildcard. This is used, for example, when determining the HTTP
1058   * address of the NN -- if it's configured to bind to 0.0.0.0, we want to
1059   * substitute the hostname from the filesystem URI rather than trying to
1060   * connect to 0.0.0.0.
1061   * @param configuredAddress the address found in the configuration
1062   * @param defaultHost the host to substitute with, if configuredAddress
1063   * is a local/wildcard address.
1064   * @return the substituted address
1065   * @throws IOException if it is a wildcard address and security is enabled
1066   */
1067  @VisibleForTesting
1068  static String substituteForWildcardAddress(String configuredAddress,
1069    String defaultHost) throws IOException {
1070    InetSocketAddress sockAddr = NetUtils.createSocketAddr(configuredAddress);
1071    InetSocketAddress defaultSockAddr = NetUtils.createSocketAddr(defaultHost
1072        + ":0");
1073    if (sockAddr.getAddress().isAnyLocalAddress()) {
1074      if (UserGroupInformation.isSecurityEnabled() &&
1075          defaultSockAddr.getAddress().isAnyLocalAddress()) {
1076        throw new IOException("Cannot use a wildcard address with security. " +
1077            "Must explicitly set bind address for Kerberos");
1078      }
1079      return defaultHost + ":" + sockAddr.getPort();
1080    } else {
1081      return configuredAddress;
1082    }
1083  }
1084  
1085  private static String getSuffixedConf(Configuration conf,
1086      String key, String defaultVal, String[] suffixes) {
1087    String ret = conf.get(DFSUtil.addKeySuffixes(key, suffixes));
1088    if (ret != null) {
1089      return ret;
1090    }
1091    return conf.get(key, defaultVal);
1092  }
1093  
1094  /**
1095   * Sets the node specific setting into generic configuration key. Looks up
1096   * value of "key.nameserviceId.namenodeId" and if found sets that value into 
1097   * generic key in the conf. If this is not found, falls back to
1098   * "key.nameserviceId" and then the unmodified key.
1099   *
1100   * Note that this only modifies the runtime conf.
1101   * 
1102   * @param conf
1103   *          Configuration object to lookup specific key and to set the value
1104   *          to the key passed. Note the conf object is modified.
1105   * @param nameserviceId
1106   *          nameservice Id to construct the node specific key. Pass null if
1107   *          federation is not configuration.
1108   * @param nnId
1109   *          namenode Id to construct the node specific key. Pass null if
1110   *          HA is not configured.
1111   * @param keys
1112   *          The key for which node specific value is looked up
1113   */
1114  public static void setGenericConf(Configuration conf,
1115      String nameserviceId, String nnId, String... keys) {
1116    for (String key : keys) {
1117      String value = conf.get(addKeySuffixes(key, nameserviceId, nnId));
1118      if (value != null) {
1119        conf.set(key, value);
1120        continue;
1121      }
1122      value = conf.get(addKeySuffixes(key, nameserviceId));
1123      if (value != null) {
1124        conf.set(key, value);
1125      }
1126    }
1127  }
1128  
1129  /** Return used as percentage of capacity */
1130  public static float getPercentUsed(long used, long capacity) {
1131    return capacity <= 0 ? 100 : (used * 100.0f)/capacity; 
1132  }
1133  
1134  /** Return remaining as percentage of capacity */
1135  public static float getPercentRemaining(long remaining, long capacity) {
1136    return capacity <= 0 ? 0 : (remaining * 100.0f)/capacity; 
1137  }
1138
1139  /** Convert percentage to a string. */
1140  public static String percent2String(double percentage) {
1141    return StringUtils.format("%.2f%%", percentage);
1142  }
1143
1144  /**
1145   * Round bytes to GiB (gibibyte)
1146   * @param bytes number of bytes
1147   * @return number of GiB
1148   */
1149  public static int roundBytesToGB(long bytes) {
1150    return Math.round((float)bytes/ 1024 / 1024 / 1024);
1151  }
1152  
1153  /** Create a {@link ClientDatanodeProtocol} proxy */
1154  public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
1155      DatanodeID datanodeid, Configuration conf, int socketTimeout,
1156      boolean connectToDnViaHostname, LocatedBlock locatedBlock) throws IOException {
1157    return new ClientDatanodeProtocolTranslatorPB(datanodeid, conf, socketTimeout,
1158        connectToDnViaHostname, locatedBlock);
1159  }
1160  
1161  /** Create {@link ClientDatanodeProtocol} proxy using kerberos ticket */
1162  static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
1163      DatanodeID datanodeid, Configuration conf, int socketTimeout,
1164      boolean connectToDnViaHostname) throws IOException {
1165    return new ClientDatanodeProtocolTranslatorPB(
1166        datanodeid, conf, socketTimeout, connectToDnViaHostname);
1167  }
1168  
1169  /** Create a {@link ClientDatanodeProtocol} proxy */
1170  public static ClientDatanodeProtocol createClientDatanodeProtocolProxy(
1171      InetSocketAddress addr, UserGroupInformation ticket, Configuration conf,
1172      SocketFactory factory) throws IOException {
1173    return new ClientDatanodeProtocolTranslatorPB(addr, ticket, conf, factory);
1174  }
1175
1176  /**
1177   * Get nameservice Id for the {@link NameNode} based on namenode RPC address
1178   * matching the local node address.
1179   */
1180  public static String getNamenodeNameServiceId(Configuration conf) {
1181    return getNameServiceId(conf, DFS_NAMENODE_RPC_ADDRESS_KEY);
1182  }
1183  
1184  /**
1185   * Get nameservice Id for the BackupNode based on backup node RPC address
1186   * matching the local node address.
1187   */
1188  public static String getBackupNameServiceId(Configuration conf) {
1189    return getNameServiceId(conf, DFS_NAMENODE_BACKUP_ADDRESS_KEY);
1190  }
1191  
1192  /**
1193   * Get nameservice Id for the secondary node based on secondary http address
1194   * matching the local node address.
1195   */
1196  public static String getSecondaryNameServiceId(Configuration conf) {
1197    return getNameServiceId(conf, DFS_NAMENODE_SECONDARY_HTTP_ADDRESS_KEY);
1198  }
1199  
1200  /**
1201   * Get the nameservice Id by matching the {@code addressKey} with the
1202   * the address of the local node. 
1203   * 
1204   * If {@link DFSConfigKeys#DFS_NAMESERVICE_ID} is not specifically
1205   * configured, and more than one nameservice Id is configured, this method 
1206   * determines the nameservice Id by matching the local node's address with the
1207   * configured addresses. When a match is found, it returns the nameservice Id
1208   * from the corresponding configuration key.
1209   * 
1210   * @param conf Configuration
1211   * @param addressKey configuration key to get the address.
1212   * @return nameservice Id on success, null if federation is not configured.
1213   * @throws HadoopIllegalArgumentException on error
1214   */
1215  private static String getNameServiceId(Configuration conf, String addressKey) {
1216    String nameserviceId = conf.get(DFS_NAMESERVICE_ID);
1217    if (nameserviceId != null) {
1218      return nameserviceId;
1219    }
1220    Collection<String> nsIds = getNameServiceIds(conf);
1221    if (1 == nsIds.size()) {
1222      return nsIds.toArray(new String[1])[0];
1223    }
1224    String nnId = conf.get(DFS_HA_NAMENODE_ID_KEY);
1225    
1226    return getSuffixIDs(conf, addressKey, null, nnId, LOCAL_ADDRESS_MATCHER)[0];
1227  }
1228  
1229  /**
1230   * Returns nameservice Id and namenode Id when the local host matches the
1231   * configuration parameter {@code addressKey}.<nameservice Id>.<namenode Id>
1232   * 
1233   * @param conf Configuration
1234   * @param addressKey configuration key corresponding to the address.
1235   * @param knownNsId only look at configs for the given nameservice, if not-null
1236   * @param knownNNId only look at configs for the given namenode, if not null
1237   * @param matcher matching criteria for matching the address
1238   * @return Array with nameservice Id and namenode Id on success. First element
1239   *         in the array is nameservice Id and second element is namenode Id.
1240   *         Null value indicates that the configuration does not have the the
1241   *         Id.
1242   * @throws HadoopIllegalArgumentException on error
1243   */
1244  static String[] getSuffixIDs(final Configuration conf, final String addressKey,
1245      String knownNsId, String knownNNId,
1246      final AddressMatcher matcher) {
1247    String nameserviceId = null;
1248    String namenodeId = null;
1249    int found = 0;
1250    
1251    Collection<String> nsIds = getNameServiceIds(conf);
1252    for (String nsId : emptyAsSingletonNull(nsIds)) {
1253      if (knownNsId != null && !knownNsId.equals(nsId)) {
1254        continue;
1255      }
1256      
1257      Collection<String> nnIds = getNameNodeIds(conf, nsId);
1258      for (String nnId : emptyAsSingletonNull(nnIds)) {
1259        if (LOG.isTraceEnabled()) {
1260          LOG.trace(String.format("addressKey: %s nsId: %s nnId: %s",
1261              addressKey, nsId, nnId));
1262        }
1263        if (knownNNId != null && !knownNNId.equals(nnId)) {
1264          continue;
1265        }
1266        String key = addKeySuffixes(addressKey, nsId, nnId);
1267        String addr = conf.get(key);
1268        if (addr == null) {
1269          continue;
1270        }
1271        InetSocketAddress s = null;
1272        try {
1273          s = NetUtils.createSocketAddr(addr);
1274        } catch (Exception e) {
1275          LOG.warn("Exception in creating socket address " + addr, e);
1276          continue;
1277        }
1278        if (!s.isUnresolved() && matcher.match(s)) {
1279          nameserviceId = nsId;
1280          namenodeId = nnId;
1281          found++;
1282        }
1283      }
1284    }
1285    if (found > 1) { // Only one address must match the local address
1286      String msg = "Configuration has multiple addresses that match "
1287          + "local node's address. Please configure the system with "
1288          + DFS_NAMESERVICE_ID + " and "
1289          + DFS_HA_NAMENODE_ID_KEY;
1290      throw new HadoopIllegalArgumentException(msg);
1291    }
1292    return new String[] { nameserviceId, namenodeId };
1293  }
1294  
1295  /**
1296   * For given set of {@code keys} adds nameservice Id and or namenode Id
1297   * and returns {nameserviceId, namenodeId} when address match is found.
1298   * @see #getSuffixIDs(Configuration, String, AddressMatcher)
1299   */
1300  static String[] getSuffixIDs(final Configuration conf,
1301      final InetSocketAddress address, final String... keys) {
1302    AddressMatcher matcher = new AddressMatcher() {
1303     @Override
1304      public boolean match(InetSocketAddress s) {
1305        return address.equals(s);
1306      } 
1307    };
1308    
1309    for (String key : keys) {
1310      String[] ids = getSuffixIDs(conf, key, null, null, matcher);
1311      if (ids != null && (ids [0] != null || ids[1] != null)) {
1312        return ids;
1313      }
1314    }
1315    return null;
1316  }
1317  
1318  private interface AddressMatcher {
1319    public boolean match(InetSocketAddress s);
1320  }
1321
1322  /** Create a URI from the scheme and address */
1323  public static URI createUri(String scheme, InetSocketAddress address) {
1324    try {
1325      return new URI(scheme, null, address.getHostName(), address.getPort(),
1326          null, null, null);
1327    } catch (URISyntaxException ue) {
1328      throw new IllegalArgumentException(ue);
1329    }
1330  }
1331  
1332  /**
1333   * Add protobuf based protocol to the {@link org.apache.hadoop.ipc.RPC.Server}
1334   * @param conf configuration
1335   * @param protocol Protocol interface
1336   * @param service service that implements the protocol
1337   * @param server RPC server to which the protocol & implementation is added to
1338   * @throws IOException
1339   */
1340  public static void addPBProtocol(Configuration conf, Class<?> protocol,
1341      BlockingService service, RPC.Server server) throws IOException {
1342    RPC.setProtocolEngine(conf, protocol, ProtobufRpcEngine.class);
1343    server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, protocol, service);
1344  }
1345
1346  /**
1347   * Map a logical namenode ID to its service address. Use the given
1348   * nameservice if specified, or the configured one if none is given.
1349   *
1350   * @param conf Configuration
1351   * @param nsId which nameservice nnId is a part of, optional
1352   * @param nnId the namenode ID to get the service addr for
1353   * @return the service addr, null if it could not be determined
1354   */
1355  public static String getNamenodeServiceAddr(final Configuration conf,
1356      String nsId, String nnId) {
1357
1358    if (nsId == null) {
1359      nsId = getOnlyNameServiceIdOrNull(conf);
1360    }
1361
1362    String serviceAddrKey = concatSuffixes(
1363        DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, nsId, nnId);
1364
1365    String addrKey = concatSuffixes(
1366        DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY, nsId, nnId);
1367
1368    String serviceRpcAddr = conf.get(serviceAddrKey);
1369    if (serviceRpcAddr == null) {
1370      serviceRpcAddr = conf.get(addrKey);
1371    }
1372    return serviceRpcAddr;
1373  }
1374
1375  /**
1376   * If the configuration refers to only a single nameservice, return the
1377   * name of that nameservice. If it refers to 0 or more than 1, return null.
1378   */
1379  public static String getOnlyNameServiceIdOrNull(Configuration conf) {
1380    Collection<String> nsIds = getNameServiceIds(conf);
1381    if (1 == nsIds.size()) {
1382      return nsIds.toArray(new String[1])[0];
1383    } else {
1384      // No nameservice ID was given and more than one is configured
1385      return null;
1386    }
1387  }
1388  
1389  public static Options helpOptions = new Options();
1390  public static Option helpOpt = new Option("h", "help", false,
1391      "get help information");
1392
1393  static {
1394    helpOptions.addOption(helpOpt);
1395  }
1396
1397  /**
1398   * Parse the arguments for commands
1399   * 
1400   * @param args the argument to be parsed
1401   * @param helpDescription help information to be printed out
1402   * @param out Printer
1403   * @param printGenericCommandUsage whether to print the 
1404   *              generic command usage defined in ToolRunner
1405   * @return true when the argument matches help option, false if not
1406   */
1407  public static boolean parseHelpArgument(String[] args,
1408      String helpDescription, PrintStream out, boolean printGenericCommandUsage) {
1409    if (args.length == 1) {
1410      try {
1411        CommandLineParser parser = new PosixParser();
1412        CommandLine cmdLine = parser.parse(helpOptions, args);
1413        if (cmdLine.hasOption(helpOpt.getOpt())
1414            || cmdLine.hasOption(helpOpt.getLongOpt())) {
1415          // should print out the help information
1416          out.println(helpDescription + "\n");
1417          if (printGenericCommandUsage) {
1418            ToolRunner.printGenericCommandUsage(out);
1419          }
1420          return true;
1421        }
1422      } catch (ParseException pe) {
1423        return false;
1424      }
1425    }
1426    return false;
1427  }
1428  
1429  /**
1430   * Get DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION from configuration.
1431   * 
1432   * @param conf Configuration
1433   * @return Value of DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION
1434   */
1435  public static float getInvalidateWorkPctPerIteration(Configuration conf) {
1436    float blocksInvalidateWorkPct = conf.getFloat(
1437        DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION,
1438        DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT);
1439    Preconditions.checkArgument(
1440        (blocksInvalidateWorkPct > 0 && blocksInvalidateWorkPct <= 1.0f),
1441        DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION +
1442        " = '" + blocksInvalidateWorkPct + "' is invalid. " +
1443        "It should be a positive, non-zero float value, not greater than 1.0f, " +
1444        "to indicate a percentage.");
1445    return blocksInvalidateWorkPct;
1446  }
1447
1448  /**
1449   * Get DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION from
1450   * configuration.
1451   * 
1452   * @param conf Configuration
1453   * @return Value of DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION
1454   */
1455  public static int getReplWorkMultiplier(Configuration conf) {
1456    int blocksReplWorkMultiplier = conf.getInt(
1457            DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,
1458            DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT);
1459    Preconditions.checkArgument(
1460        (blocksReplWorkMultiplier > 0),
1461        DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION +
1462        " = '" + blocksReplWorkMultiplier + "' is invalid. " +
1463        "It should be a positive, non-zero integer value.");
1464    return blocksReplWorkMultiplier;
1465  }
1466  
1467  /**
1468   * Get SPNEGO keytab Key from configuration
1469   * 
1470   * @param conf
1471   *          Configuration
1472   * @param defaultKey
1473   * @return DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY if the key is not empty
1474   *         else return defaultKey
1475   */
1476  public static String getSpnegoKeytabKey(Configuration conf, String defaultKey) {
1477    String value = 
1478        conf.get(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY);
1479    return (value == null || value.isEmpty()) ?
1480        defaultKey : DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY;
1481  }
1482
1483  /**
1484   * Get http policy. Http Policy is chosen as follows:
1485   * <ol>
1486   * <li>If hadoop.ssl.enabled is set, http endpoints are not started. Only
1487   * https endpoints are started on configured https ports</li>
1488   * <li>This configuration is overridden by dfs.https.enable configuration, if
1489   * it is set to true. In that case, both http and https endpoints are stared.</li>
1490   * <li>All the above configurations are overridden by dfs.http.policy
1491   * configuration. With this configuration you can set http-only, https-only
1492   * and http-and-https endpoints.</li>
1493   * </ol>
1494   * See hdfs-default.xml documentation for more details on each of the above
1495   * configuration settings.
1496   */
1497  public static HttpConfig.Policy getHttpPolicy(Configuration conf) {
1498    String policyStr = conf.get(DFSConfigKeys.DFS_HTTP_POLICY_KEY);
1499    if (policyStr == null) {
1500      boolean https = conf.getBoolean(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY,
1501          DFSConfigKeys.DFS_HTTPS_ENABLE_DEFAULT);
1502
1503      boolean hadoopSsl = conf.getBoolean(
1504          CommonConfigurationKeys.HADOOP_SSL_ENABLED_KEY,
1505          CommonConfigurationKeys.HADOOP_SSL_ENABLED_DEFAULT);
1506
1507      if (hadoopSsl) {
1508        LOG.warn(CommonConfigurationKeys.HADOOP_SSL_ENABLED_KEY
1509            + " is deprecated. Please use " + DFSConfigKeys.DFS_HTTP_POLICY_KEY
1510            + ".");
1511      }
1512      if (https) {
1513        LOG.warn(DFSConfigKeys.DFS_HTTPS_ENABLE_KEY
1514            + " is deprecated. Please use " + DFSConfigKeys.DFS_HTTP_POLICY_KEY
1515            + ".");
1516      }
1517
1518      return (hadoopSsl || https) ? HttpConfig.Policy.HTTP_AND_HTTPS
1519          : HttpConfig.Policy.HTTP_ONLY;
1520    }
1521
1522    HttpConfig.Policy policy = HttpConfig.Policy.fromString(policyStr);
1523    if (policy == null) {
1524      throw new HadoopIllegalArgumentException("Unregonized value '"
1525          + policyStr + "' for " + DFSConfigKeys.DFS_HTTP_POLICY_KEY);
1526    }
1527
1528    conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, policy.name());
1529    return policy;
1530  }
1531
1532  public static HttpServer2.Builder loadSslConfToHttpServerBuilder(HttpServer2.Builder builder,
1533      Configuration sslConf) {
1534    return builder
1535        .needsClientAuth(
1536            sslConf.getBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY,
1537                DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT))
1538        .keyPassword(sslConf.get("ssl.server.keystore.keypassword"))
1539        .keyStore(sslConf.get("ssl.server.keystore.location"),
1540            sslConf.get("ssl.server.keystore.password"),
1541            sslConf.get("ssl.server.keystore.type", "jks"))
1542        .trustStore(sslConf.get("ssl.server.truststore.location"),
1543            sslConf.get("ssl.server.truststore.password"),
1544            sslConf.get("ssl.server.truststore.type", "jks"));
1545  }
1546
1547  /**
1548   * Load HTTPS-related configuration.
1549   */
1550  public static Configuration loadSslConfiguration(Configuration conf) {
1551    Configuration sslConf = new Configuration(false);
1552
1553    sslConf.addResource(conf.get(
1554        DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY,
1555        DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_DEFAULT));
1556
1557    boolean requireClientAuth = conf.getBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY,
1558        DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT);
1559    sslConf.setBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY, requireClientAuth);
1560    return sslConf;
1561  }
1562
1563  /**
1564   * Return a HttpServer.Builder that the journalnode / namenode / secondary
1565   * namenode can use to initialize their HTTP / HTTPS server.
1566   *
1567   */
1568  public static HttpServer2.Builder httpServerTemplateForNNAndJN(
1569      Configuration conf, final InetSocketAddress httpAddr,
1570      final InetSocketAddress httpsAddr, String name, String spnegoUserNameKey,
1571      String spnegoKeytabFileKey) throws IOException {
1572    HttpConfig.Policy policy = getHttpPolicy(conf);
1573
1574    HttpServer2.Builder builder = new HttpServer2.Builder().setName(name)
1575        .setConf(conf).setACL(new AccessControlList(conf.get(DFS_ADMIN, " ")))
1576        .setSecurityEnabled(UserGroupInformation.isSecurityEnabled())
1577        .setUsernameConfKey(spnegoUserNameKey)
1578        .setKeytabConfKey(getSpnegoKeytabKey(conf, spnegoKeytabFileKey));
1579
1580    // initialize the webserver for uploading/downloading files.
1581    LOG.info("Starting web server as: "
1582        + SecurityUtil.getServerPrincipal(conf.get(spnegoUserNameKey),
1583            httpAddr.getHostName()));
1584
1585    if (policy.isHttpEnabled()) {
1586      if (httpAddr.getPort() == 0) {
1587        builder.setFindPort(true);
1588      }
1589
1590      URI uri = URI.create("http://" + NetUtils.getHostPortString(httpAddr));
1591      builder.addEndpoint(uri);
1592      LOG.info("Starting Web-server for " + name + " at: " + uri);
1593    }
1594
1595    if (policy.isHttpsEnabled() && httpsAddr != null) {
1596      Configuration sslConf = loadSslConfiguration(conf);
1597      loadSslConfToHttpServerBuilder(builder, sslConf);
1598
1599      if (httpsAddr.getPort() == 0) {
1600        builder.setFindPort(true);
1601      }
1602
1603      URI uri = URI.create("https://" + NetUtils.getHostPortString(httpsAddr));
1604      builder.addEndpoint(uri);
1605      LOG.info("Starting Web-server for " + name + " at: " + uri);
1606    }
1607    return builder;
1608  }
1609
1610  /**
1611   * Converts a Date into an ISO-8601 formatted datetime string.
1612   */
1613  public static String dateToIso8601String(Date date) {
1614    SimpleDateFormat df =
1615        new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.ENGLISH);
1616    return df.format(date);
1617  }
1618
1619  /**
1620   * Converts a time duration in milliseconds into DDD:HH:MM:SS format.
1621   */
1622  public static String durationToString(long durationMs) {
1623    boolean negative = false;
1624    if (durationMs < 0) {
1625      negative = true;
1626      durationMs = -durationMs;
1627    }
1628    // Chop off the milliseconds
1629    long durationSec = durationMs / 1000;
1630    final int secondsPerMinute = 60;
1631    final int secondsPerHour = 60*60;
1632    final int secondsPerDay = 60*60*24;
1633    final long days = durationSec / secondsPerDay;
1634    durationSec -= days * secondsPerDay;
1635    final long hours = durationSec / secondsPerHour;
1636    durationSec -= hours * secondsPerHour;
1637    final long minutes = durationSec / secondsPerMinute;
1638    durationSec -= minutes * secondsPerMinute;
1639    final long seconds = durationSec;
1640    final long milliseconds = durationMs % 1000;
1641    String format = "%03d:%02d:%02d:%02d.%03d";
1642    if (negative)  {
1643      format = "-" + format;
1644    }
1645    return String.format(format, days, hours, minutes, seconds, milliseconds);
1646  }
1647
1648  /**
1649   * Converts a relative time string into a duration in milliseconds.
1650   */
1651  public static long parseRelativeTime(String relTime) throws IOException {
1652    if (relTime.length() < 2) {
1653      throw new IOException("Unable to parse relative time value of " + relTime
1654          + ": too short");
1655    }
1656    String ttlString = relTime.substring(0, relTime.length()-1);
1657    long ttl;
1658    try {
1659      ttl = Long.parseLong(ttlString);
1660    } catch (NumberFormatException e) {
1661      throw new IOException("Unable to parse relative time value of " + relTime
1662          + ": " + ttlString + " is not a number");
1663    }
1664    if (relTime.endsWith("s")) {
1665      // pass
1666    } else if (relTime.endsWith("m")) {
1667      ttl *= 60;
1668    } else if (relTime.endsWith("h")) {
1669      ttl *= 60*60;
1670    } else if (relTime.endsWith("d")) {
1671      ttl *= 60*60*24;
1672    } else {
1673      throw new IOException("Unable to parse relative time value of " + relTime
1674          + ": unknown time unit " + relTime.charAt(relTime.length() - 1));
1675    }
1676    return ttl*1000;
1677  }
1678}