001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs;
019
020import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT;
021import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY;
022import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT;
023import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY;
024import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_DEFAULT;
025import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY;
026import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_RETRIES_DEFAULT;
027import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_BLOCK_WRITE_RETRIES_KEY;
028import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHED_CONN_RETRY_DEFAULT;
029import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHED_CONN_RETRY_KEY;
030import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_READS;
031import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_WRITES;
032import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_READAHEAD;
033import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT;
034import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY;
035import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT;
036import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY;
037import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT;
038import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY;
039import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT;
040import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY;
041import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT;
042import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY;
043import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY;
044import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE;
045import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT;
046import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY;
047import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT;
048import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY;
049import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
050import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME;
051import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT;
052import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL;
053import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_DEFAULT;
054import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT;
055import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY;
056import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY;
057import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
058import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
059
060import java.io.BufferedOutputStream;
061import java.io.DataInputStream;
062import java.io.DataOutputStream;
063import java.io.FileNotFoundException;
064import java.io.IOException;
065import java.io.InputStream;
066import java.io.OutputStream;
067import java.net.InetAddress;
068import java.net.InetSocketAddress;
069import java.net.NetworkInterface;
070import java.net.Socket;
071import java.net.SocketException;
072import java.net.SocketAddress;
073import java.net.URI;
074import java.net.UnknownHostException;
075import java.util.ArrayList;
076import java.util.Collections;
077import java.util.EnumSet;
078import java.util.HashMap;
079import java.util.LinkedHashMap;
080import java.util.List;
081import java.util.Map;
082import java.util.Random;
083
084import javax.net.SocketFactory;
085
086import org.apache.commons.logging.Log;
087import org.apache.commons.logging.LogFactory;
088import org.apache.hadoop.classification.InterfaceAudience;
089import org.apache.hadoop.conf.Configuration;
090import org.apache.hadoop.fs.BlockLocation;
091import org.apache.hadoop.fs.BlockStorageLocation;
092import org.apache.hadoop.fs.CacheFlag;
093import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
094import org.apache.hadoop.fs.ContentSummary;
095import org.apache.hadoop.fs.CreateFlag;
096import org.apache.hadoop.fs.FileAlreadyExistsException;
097import org.apache.hadoop.fs.FileSystem;
098import org.apache.hadoop.fs.FsServerDefaults;
099import org.apache.hadoop.fs.FsStatus;
100import org.apache.hadoop.fs.HdfsBlockLocation;
101import org.apache.hadoop.fs.InvalidPathException;
102import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum;
103import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
104import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum;
105import org.apache.hadoop.fs.Options;
106import org.apache.hadoop.fs.RemoteIterator;
107import org.apache.hadoop.fs.Options.ChecksumOpt;
108import org.apache.hadoop.fs.ParentNotDirectoryException;
109import org.apache.hadoop.fs.Path;
110import org.apache.hadoop.fs.UnresolvedLinkException;
111import org.apache.hadoop.fs.VolumeId;
112import org.apache.hadoop.fs.permission.FsPermission;
113import org.apache.hadoop.hdfs.client.ClientMmapManager;
114import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
115import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
116import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
117import org.apache.hadoop.hdfs.protocol.CacheDirectiveIterator;
118import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
119import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
120import org.apache.hadoop.hdfs.protocol.CachePoolIterator;
121import org.apache.hadoop.hdfs.protocol.ClientProtocol;
122import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
123import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
124import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
125import org.apache.hadoop.hdfs.protocol.DirectoryListing;
126import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
127import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata;
128import org.apache.hadoop.hdfs.protocol.HdfsConstants;
129import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
130import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
131import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
132import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
133import org.apache.hadoop.hdfs.protocol.LocatedBlock;
134import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
135import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
136import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
137import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
138import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
139import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
140import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferEncryptor;
141import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
142import org.apache.hadoop.hdfs.protocol.datatransfer.Op;
143import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure;
144import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
145import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
146import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpBlockChecksumResponseProto;
147import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
148import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
149import org.apache.hadoop.hdfs.protocolPB.PBHelper;
150import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
151import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
152import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
153import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
154import org.apache.hadoop.hdfs.server.namenode.NameNode;
155import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
156import org.apache.hadoop.io.DataOutputBuffer;
157import org.apache.hadoop.io.EnumSetWritable;
158import org.apache.hadoop.io.IOUtils;
159import org.apache.hadoop.io.MD5Hash;
160import org.apache.hadoop.io.Text;
161import org.apache.hadoop.io.retry.LossyRetryInvocationHandler;
162import org.apache.hadoop.ipc.Client;
163import org.apache.hadoop.ipc.RPC;
164import org.apache.hadoop.ipc.RemoteException;
165import org.apache.hadoop.net.DNS;
166import org.apache.hadoop.net.NetUtils;
167import org.apache.hadoop.security.AccessControlException;
168import org.apache.hadoop.security.UserGroupInformation;
169import org.apache.hadoop.security.token.SecretManager.InvalidToken;
170import org.apache.hadoop.security.token.Token;
171import org.apache.hadoop.security.token.TokenRenewer;
172import org.apache.hadoop.util.DataChecksum;
173import org.apache.hadoop.util.DataChecksum.Type;
174import org.apache.hadoop.util.Progressable;
175import org.apache.hadoop.util.Time;
176
177import com.google.common.annotations.VisibleForTesting;
178import com.google.common.base.Joiner;
179import com.google.common.base.Preconditions;
180import com.google.common.net.InetAddresses;
181
182/********************************************************
183 * DFSClient can connect to a Hadoop Filesystem and 
184 * perform basic file tasks.  It uses the ClientProtocol
185 * to communicate with a NameNode daemon, and connects 
186 * directly to DataNodes to read/write block data.
187 *
188 * Hadoop DFS users should obtain an instance of 
189 * DistributedFileSystem, which uses DFSClient to handle
190 * filesystem tasks.
191 *
192 ********************************************************/
193@InterfaceAudience.Private
194public class DFSClient implements java.io.Closeable {
195  public static final Log LOG = LogFactory.getLog(DFSClient.class);
196  public static final long SERVER_DEFAULTS_VALIDITY_PERIOD = 60 * 60 * 1000L; // 1 hour
197  static final int TCP_WINDOW_SIZE = 128 * 1024; // 128 KB
198
199  private final Configuration conf;
200  private final Conf dfsClientConf;
201  final ClientProtocol namenode;
202  /* The service used for delegation tokens */
203  private Text dtService;
204
205  final UserGroupInformation ugi;
206  volatile boolean clientRunning = true;
207  volatile long lastLeaseRenewal;
208  private volatile FsServerDefaults serverDefaults;
209  private volatile long serverDefaultsLastUpdate;
210  final String clientName;
211  SocketFactory socketFactory;
212  final ReplaceDatanodeOnFailure dtpReplaceDatanodeOnFailure;
213  final FileSystem.Statistics stats;
214  private final String authority;
215  final PeerCache peerCache;
216  private Random r = new Random();
217  private SocketAddress[] localInterfaceAddrs;
218  private DataEncryptionKey encryptionKey;
219  private boolean shouldUseLegacyBlockReaderLocal;
220  private final CachingStrategy defaultReadCachingStrategy;
221  private final CachingStrategy defaultWriteCachingStrategy;
222  private ClientMmapManager mmapManager;
223  
224  private static final ClientMmapManagerFactory MMAP_MANAGER_FACTORY =
225      new ClientMmapManagerFactory();
226
227  private static final class ClientMmapManagerFactory {
228    private ClientMmapManager mmapManager = null;
229    /**
230     * Tracks the number of users of mmapManager.
231     */
232    private int refcnt = 0;
233
234    synchronized ClientMmapManager get(Configuration conf) {
235      if (refcnt++ == 0) {
236        mmapManager = ClientMmapManager.fromConf(conf);
237      } else {
238        String mismatches = mmapManager.verifyConfigurationMatches(conf);
239        if (!mismatches.isEmpty()) {
240          LOG.warn("The ClientMmapManager settings you specified " +
241            "have been ignored because another thread created the " +
242            "ClientMmapManager first.  " + mismatches);
243        }
244      }
245      return mmapManager;
246    }
247    
248    synchronized void unref(ClientMmapManager mmapManager) {
249      if (this.mmapManager != mmapManager) {
250        throw new IllegalArgumentException();
251      }
252      if (--refcnt == 0) {
253        IOUtils.cleanup(LOG, mmapManager);
254        mmapManager = null;
255      }
256    }
257  }
258
259  /**
260   * DFSClient configuration 
261   */
262  public static class Conf {
263    final int hdfsTimeout;    // timeout value for a DFS operation.
264    final int maxFailoverAttempts;
265    final int maxRetryAttempts;
266    final int failoverSleepBaseMillis;
267    final int failoverSleepMaxMillis;
268    final int maxBlockAcquireFailures;
269    final int confTime;
270    final int ioBufferSize;
271    final ChecksumOpt defaultChecksumOpt;
272    final int writePacketSize;
273    final int socketTimeout;
274    final int socketCacheCapacity;
275    final long socketCacheExpiry;
276    final long excludedNodesCacheExpiry;
277    /** Wait time window (in msec) if BlockMissingException is caught */
278    final int timeWindow;
279    final int nCachedConnRetry;
280    final int nBlockWriteRetry;
281    final int nBlockWriteLocateFollowingRetry;
282    final long defaultBlockSize;
283    final long prefetchSize;
284    final short defaultReplication;
285    final String taskId;
286    final FsPermission uMask;
287    final boolean connectToDnViaHostname;
288    final boolean getHdfsBlocksMetadataEnabled;
289    final int getFileBlockStorageLocationsNumThreads;
290    final int getFileBlockStorageLocationsTimeout;
291    final int retryTimesForGetLastBlockLength;
292    final int retryIntervalForGetLastBlockLength;
293
294    final boolean useLegacyBlockReader;
295    final boolean useLegacyBlockReaderLocal;
296    final String domainSocketPath;
297    final boolean skipShortCircuitChecksums;
298    final int shortCircuitBufferSize;
299    final boolean shortCircuitLocalReads;
300    final boolean domainSocketDataTraffic;
301    final int shortCircuitStreamsCacheSize;
302    final long shortCircuitStreamsCacheExpiryMs; 
303
304    public Conf(Configuration conf) {
305      // The hdfsTimeout is currently the same as the ipc timeout 
306      hdfsTimeout = Client.getTimeout(conf);
307
308      maxFailoverAttempts = conf.getInt(
309          DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY,
310          DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT);
311      maxRetryAttempts = conf.getInt(
312          DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY,
313          DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT);
314      failoverSleepBaseMillis = conf.getInt(
315          DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY,
316          DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT);
317      failoverSleepMaxMillis = conf.getInt(
318          DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY,
319          DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT);
320
321      maxBlockAcquireFailures = conf.getInt(
322          DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY,
323          DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT);
324      confTime = conf.getInt(DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY,
325          HdfsServerConstants.WRITE_TIMEOUT);
326      ioBufferSize = conf.getInt(
327          CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
328          CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
329      defaultChecksumOpt = getChecksumOptFromConf(conf);
330      socketTimeout = conf.getInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY,
331          HdfsServerConstants.READ_TIMEOUT);
332      /** dfs.write.packet.size is an internal config variable */
333      writePacketSize = conf.getInt(DFS_CLIENT_WRITE_PACKET_SIZE_KEY,
334          DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT);
335      defaultBlockSize = conf.getLongBytes(DFS_BLOCK_SIZE_KEY,
336          DFS_BLOCK_SIZE_DEFAULT);
337      defaultReplication = (short) conf.getInt(
338          DFS_REPLICATION_KEY, DFS_REPLICATION_DEFAULT);
339      taskId = conf.get("mapreduce.task.attempt.id", "NONMAPREDUCE");
340      socketCacheCapacity = conf.getInt(DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY,
341          DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT);
342      socketCacheExpiry = conf.getLong(DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY,
343          DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_DEFAULT);
344      excludedNodesCacheExpiry = conf.getLong(
345          DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL,
346          DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL_DEFAULT);
347      prefetchSize = conf.getLong(DFS_CLIENT_READ_PREFETCH_SIZE_KEY,
348          10 * defaultBlockSize);
349      timeWindow = conf.getInt(DFS_CLIENT_RETRY_WINDOW_BASE, 3000);
350      nCachedConnRetry = conf.getInt(DFS_CLIENT_CACHED_CONN_RETRY_KEY,
351          DFS_CLIENT_CACHED_CONN_RETRY_DEFAULT);
352      nBlockWriteRetry = conf.getInt(DFS_CLIENT_BLOCK_WRITE_RETRIES_KEY,
353          DFS_CLIENT_BLOCK_WRITE_RETRIES_DEFAULT);
354      nBlockWriteLocateFollowingRetry = conf.getInt(
355          DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_KEY,
356          DFS_CLIENT_BLOCK_WRITE_LOCATEFOLLOWINGBLOCK_RETRIES_DEFAULT);
357      uMask = FsPermission.getUMask(conf);
358      connectToDnViaHostname = conf.getBoolean(DFS_CLIENT_USE_DN_HOSTNAME,
359          DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT);
360      getHdfsBlocksMetadataEnabled = conf.getBoolean(
361          DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, 
362          DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT);
363      getFileBlockStorageLocationsNumThreads = conf.getInt(
364          DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_NUM_THREADS,
365          DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_NUM_THREADS_DEFAULT);
366      getFileBlockStorageLocationsTimeout = conf.getInt(
367          DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT,
368          DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT_DEFAULT);
369      retryTimesForGetLastBlockLength = conf.getInt(
370          DFSConfigKeys.DFS_CLIENT_RETRY_TIMES_GET_LAST_BLOCK_LENGTH,
371          DFSConfigKeys.DFS_CLIENT_RETRY_TIMES_GET_LAST_BLOCK_LENGTH_DEFAULT);
372      retryIntervalForGetLastBlockLength = conf.getInt(
373        DFSConfigKeys.DFS_CLIENT_RETRY_INTERVAL_GET_LAST_BLOCK_LENGTH,
374        DFSConfigKeys.DFS_CLIENT_RETRY_INTERVAL_GET_LAST_BLOCK_LENGTH_DEFAULT);
375
376      useLegacyBlockReader = conf.getBoolean(
377          DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADER,
378          DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADER_DEFAULT);
379      useLegacyBlockReaderLocal = conf.getBoolean(
380          DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL,
381          DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL_DEFAULT);
382      shortCircuitLocalReads = conf.getBoolean(
383          DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY,
384          DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_DEFAULT);
385      domainSocketDataTraffic = conf.getBoolean(
386          DFSConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC,
387          DFSConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC_DEFAULT);
388      domainSocketPath = conf.getTrimmed(
389          DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
390          DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_DEFAULT);
391
392      if (BlockReaderLocal.LOG.isDebugEnabled()) {
393        BlockReaderLocal.LOG.debug(
394            DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL
395            + " = " + useLegacyBlockReaderLocal);
396        BlockReaderLocal.LOG.debug(
397            DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY
398            + " = " + shortCircuitLocalReads);
399        BlockReaderLocal.LOG.debug(
400            DFSConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC
401            + " = " + domainSocketDataTraffic);
402        BlockReaderLocal.LOG.debug(
403            DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY
404            + " = " + domainSocketPath);
405      }
406
407      skipShortCircuitChecksums = conf.getBoolean(
408          DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY,
409          DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_DEFAULT);
410      shortCircuitBufferSize = conf.getInt(
411          DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_BUFFER_SIZE_KEY,
412          DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_BUFFER_SIZE_DEFAULT);
413      shortCircuitStreamsCacheSize = conf.getInt(
414          DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_SIZE_KEY,
415          DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_SIZE_DEFAULT);
416      shortCircuitStreamsCacheExpiryMs = conf.getLong(
417          DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_KEY,
418          DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_DEFAULT);
419    }
420
421    private DataChecksum.Type getChecksumType(Configuration conf) {
422      final String checksum = conf.get(
423          DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY,
424          DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT);
425      try {
426        return DataChecksum.Type.valueOf(checksum);
427      } catch(IllegalArgumentException iae) {
428        LOG.warn("Bad checksum type: " + checksum + ". Using default "
429            + DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT);
430        return DataChecksum.Type.valueOf(
431            DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT); 
432      }
433    }
434
435    // Construct a checksum option from conf
436    private ChecksumOpt getChecksumOptFromConf(Configuration conf) {
437      DataChecksum.Type type = getChecksumType(conf);
438      int bytesPerChecksum = conf.getInt(DFS_BYTES_PER_CHECKSUM_KEY,
439          DFS_BYTES_PER_CHECKSUM_DEFAULT);
440      return new ChecksumOpt(type, bytesPerChecksum);
441    }
442
443    // create a DataChecksum with the default option.
444    private DataChecksum createChecksum() throws IOException {
445      return createChecksum(null);
446    }
447
448    private DataChecksum createChecksum(ChecksumOpt userOpt) 
449        throws IOException {
450      // Fill in any missing field with the default.
451      ChecksumOpt myOpt = ChecksumOpt.processChecksumOpt(
452          defaultChecksumOpt, userOpt);
453      DataChecksum dataChecksum = DataChecksum.newDataChecksum(
454          myOpt.getChecksumType(),
455          myOpt.getBytesPerChecksum());
456      if (dataChecksum == null) {
457        throw new IOException("Invalid checksum type specified: "
458            + myOpt.getChecksumType().name());
459      }
460      return dataChecksum;
461    }
462  }
463 
464  public Conf getConf() {
465    return dfsClientConf;
466  }
467  
468  Configuration getConfiguration() {
469    return conf;
470  }
471  
472  /**
473   * A map from file names to {@link DFSOutputStream} objects
474   * that are currently being written by this client.
475   * Note that a file can only be written by a single client.
476   */
477  private final Map<String, DFSOutputStream> filesBeingWritten
478      = new HashMap<String, DFSOutputStream>();
479
480  private final DomainSocketFactory domainSocketFactory;
481  
482  /**
483   * Same as this(NameNode.getAddress(conf), conf);
484   * @see #DFSClient(InetSocketAddress, Configuration)
485   * @deprecated Deprecated at 0.21
486   */
487  @Deprecated
488  public DFSClient(Configuration conf) throws IOException {
489    this(NameNode.getAddress(conf), conf);
490  }
491  
492  public DFSClient(InetSocketAddress address, Configuration conf) throws IOException {
493    this(NameNode.getUri(address), conf);
494  }
495
496  /**
497   * Same as this(nameNodeUri, conf, null);
498   * @see #DFSClient(URI, Configuration, FileSystem.Statistics)
499   */
500  public DFSClient(URI nameNodeUri, Configuration conf
501      ) throws IOException {
502    this(nameNodeUri, conf, null);
503  }
504
505  /**
506   * Same as this(nameNodeUri, null, conf, stats);
507   * @see #DFSClient(URI, ClientProtocol, Configuration, FileSystem.Statistics) 
508   */
509  public DFSClient(URI nameNodeUri, Configuration conf,
510                   FileSystem.Statistics stats)
511    throws IOException {
512    this(nameNodeUri, null, conf, stats);
513  }
514  
515  /** 
516   * Create a new DFSClient connected to the given nameNodeUri or rpcNamenode.
517   * If HA is enabled and a positive value is set for 
518   * {@link DFSConfigKeys#DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY} in the
519   * configuration, the DFSClient will use {@link LossyRetryInvocationHandler}
520   * as its RetryInvocationHandler. Otherwise one of nameNodeUri or rpcNamenode 
521   * must be null.
522   */
523  @VisibleForTesting
524  public DFSClient(URI nameNodeUri, ClientProtocol rpcNamenode,
525      Configuration conf, FileSystem.Statistics stats)
526    throws IOException {
527    // Copy only the required DFSClient configuration
528    this.dfsClientConf = new Conf(conf);
529    this.shouldUseLegacyBlockReaderLocal = 
530        this.dfsClientConf.useLegacyBlockReaderLocal;
531    if (this.dfsClientConf.useLegacyBlockReaderLocal) {
532      LOG.debug("Using legacy short-circuit local reads.");
533    }
534    this.conf = conf;
535    this.stats = stats;
536    this.socketFactory = NetUtils.getSocketFactory(conf, ClientProtocol.class);
537    this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf);
538
539    this.ugi = UserGroupInformation.getCurrentUser();
540    
541    this.authority = nameNodeUri == null? "null": nameNodeUri.getAuthority();
542    this.clientName = "DFSClient_" + dfsClientConf.taskId + "_" + 
543        DFSUtil.getRandom().nextInt()  + "_" + Thread.currentThread().getId();
544    
545    int numResponseToDrop = conf.getInt(
546        DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY,
547        DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT);
548    NameNodeProxies.ProxyAndInfo<ClientProtocol> proxyInfo = null;
549    if (numResponseToDrop > 0) {
550      // This case is used for testing.
551      LOG.warn(DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY
552          + " is set to " + numResponseToDrop
553          + ", this hacked client will proactively drop responses");
554      proxyInfo = NameNodeProxies.createProxyWithLossyRetryHandler(conf,
555          nameNodeUri, ClientProtocol.class, numResponseToDrop);
556    }
557    
558    if (proxyInfo != null) {
559      this.dtService = proxyInfo.getDelegationTokenService();
560      this.namenode = proxyInfo.getProxy();
561    } else if (rpcNamenode != null) {
562      // This case is used for testing.
563      Preconditions.checkArgument(nameNodeUri == null);
564      this.namenode = rpcNamenode;
565      dtService = null;
566    } else {
567      Preconditions.checkArgument(nameNodeUri != null,
568          "null URI");
569      proxyInfo = NameNodeProxies.createProxy(conf, nameNodeUri,
570          ClientProtocol.class);
571      this.dtService = proxyInfo.getDelegationTokenService();
572      this.namenode = proxyInfo.getProxy();
573    }
574
575    // read directly from the block file if configured.
576    this.domainSocketFactory = new DomainSocketFactory(dfsClientConf);
577
578    String localInterfaces[] =
579      conf.getTrimmedStrings(DFSConfigKeys.DFS_CLIENT_LOCAL_INTERFACES);
580    localInterfaceAddrs = getLocalInterfaceAddrs(localInterfaces);
581    if (LOG.isDebugEnabled() && 0 != localInterfaces.length) {
582      LOG.debug("Using local interfaces [" +
583      Joiner.on(',').join(localInterfaces)+ "] with addresses [" +
584      Joiner.on(',').join(localInterfaceAddrs) + "]");
585    }
586    
587    this.peerCache = PeerCache.getInstance(dfsClientConf.socketCacheCapacity, dfsClientConf.socketCacheExpiry);
588    Boolean readDropBehind = (conf.get(DFS_CLIENT_CACHE_DROP_BEHIND_READS) == null) ?
589        null : conf.getBoolean(DFS_CLIENT_CACHE_DROP_BEHIND_READS, false);
590    Long readahead = (conf.get(DFS_CLIENT_CACHE_READAHEAD) == null) ?
591        null : conf.getLong(DFS_CLIENT_CACHE_READAHEAD, 0);
592    Boolean writeDropBehind = (conf.get(DFS_CLIENT_CACHE_DROP_BEHIND_WRITES) == null) ?
593        null : conf.getBoolean(DFS_CLIENT_CACHE_DROP_BEHIND_WRITES, false);
594    this.defaultReadCachingStrategy =
595        new CachingStrategy(readDropBehind, readahead);
596    this.defaultWriteCachingStrategy =
597        new CachingStrategy(writeDropBehind, readahead);
598    this.mmapManager = MMAP_MANAGER_FACTORY.get(conf);
599  }
600  
601  /**
602   * Return the socket addresses to use with each configured
603   * local interface. Local interfaces may be specified by IP
604   * address, IP address range using CIDR notation, interface
605   * name (e.g. eth0) or sub-interface name (e.g. eth0:0).
606   * The socket addresses consist of the IPs for the interfaces
607   * and the ephemeral port (port 0). If an IP, IP range, or
608   * interface name matches an interface with sub-interfaces
609   * only the IP of the interface is used. Sub-interfaces can
610   * be used by specifying them explicitly (by IP or name).
611   * 
612   * @return SocketAddresses for the configured local interfaces,
613   *    or an empty array if none are configured
614   * @throws UnknownHostException if a given interface name is invalid
615   */
616  private static SocketAddress[] getLocalInterfaceAddrs(
617      String interfaceNames[]) throws UnknownHostException {
618    List<SocketAddress> localAddrs = new ArrayList<SocketAddress>();
619    for (String interfaceName : interfaceNames) {
620      if (InetAddresses.isInetAddress(interfaceName)) {
621        localAddrs.add(new InetSocketAddress(interfaceName, 0));
622      } else if (NetUtils.isValidSubnet(interfaceName)) {
623        for (InetAddress addr : NetUtils.getIPs(interfaceName, false)) {
624          localAddrs.add(new InetSocketAddress(addr, 0));
625        }
626      } else {
627        for (String ip : DNS.getIPs(interfaceName, false)) {
628          localAddrs.add(new InetSocketAddress(ip, 0));
629        }
630      }
631    }
632    return localAddrs.toArray(new SocketAddress[localAddrs.size()]);
633  }
634
635  /**
636   * Select one of the configured local interfaces at random. We use a random
637   * interface because other policies like round-robin are less effective
638   * given that we cache connections to datanodes.
639   *
640   * @return one of the local interface addresses at random, or null if no
641   *    local interfaces are configured
642   */
643  SocketAddress getRandomLocalInterfaceAddr() {
644    if (localInterfaceAddrs.length == 0) {
645      return null;
646    }
647    final int idx = r.nextInt(localInterfaceAddrs.length);
648    final SocketAddress addr = localInterfaceAddrs[idx];
649    if (LOG.isDebugEnabled()) {
650      LOG.debug("Using local interface " + addr);
651    }
652    return addr;
653  }
654
655  /**
656   * Return the number of times the client should go back to the namenode
657   * to retrieve block locations when reading.
658   */
659  int getMaxBlockAcquireFailures() {
660    return dfsClientConf.maxBlockAcquireFailures;
661  }
662
663  /**
664   * Return the timeout that clients should use when writing to datanodes.
665   * @param numNodes the number of nodes in the pipeline.
666   */
667  int getDatanodeWriteTimeout(int numNodes) {
668    return (dfsClientConf.confTime > 0) ?
669      (dfsClientConf.confTime + HdfsServerConstants.WRITE_TIMEOUT_EXTENSION * numNodes) : 0;
670  }
671
672  int getDatanodeReadTimeout(int numNodes) {
673    return dfsClientConf.socketTimeout > 0 ?
674        (HdfsServerConstants.READ_TIMEOUT_EXTENSION * numNodes +
675            dfsClientConf.socketTimeout) : 0;
676  }
677  
678  int getHdfsTimeout() {
679    return dfsClientConf.hdfsTimeout;
680  }
681  
682  @VisibleForTesting
683  public String getClientName() {
684    return clientName;
685  }
686
687  void checkOpen() throws IOException {
688    if (!clientRunning) {
689      IOException result = new IOException("Filesystem closed");
690      throw result;
691    }
692  }
693
694  /** Return the lease renewer instance. The renewer thread won't start
695   *  until the first output stream is created. The same instance will
696   *  be returned until all output streams are closed.
697   */
698  public LeaseRenewer getLeaseRenewer() throws IOException {
699      return LeaseRenewer.getInstance(authority, ugi, this);
700  }
701
702  /** Get a lease and start automatic renewal */
703  private void beginFileLease(final String src, final DFSOutputStream out) 
704      throws IOException {
705    getLeaseRenewer().put(src, out, this);
706  }
707
708  /** Stop renewal of lease for the file. */
709  void endFileLease(final String src) throws IOException {
710    getLeaseRenewer().closeFile(src, this);
711  }
712    
713
714  /** Put a file. Only called from LeaseRenewer, where proper locking is
715   *  enforced to consistently update its local dfsclients array and 
716   *  client's filesBeingWritten map.
717   */
718  void putFileBeingWritten(final String src, final DFSOutputStream out) {
719    synchronized(filesBeingWritten) {
720      filesBeingWritten.put(src, out);
721      // update the last lease renewal time only when there was no
722      // writes. once there is one write stream open, the lease renewer
723      // thread keeps it updated well with in anyone's expiration time.
724      if (lastLeaseRenewal == 0) {
725        updateLastLeaseRenewal();
726      }
727    }
728  }
729
730  /** Remove a file. Only called from LeaseRenewer. */
731  void removeFileBeingWritten(final String src) {
732    synchronized(filesBeingWritten) {
733      filesBeingWritten.remove(src);
734      if (filesBeingWritten.isEmpty()) {
735        lastLeaseRenewal = 0;
736      }
737    }
738  }
739
740  /** Is file-being-written map empty? */
741  boolean isFilesBeingWrittenEmpty() {
742    synchronized(filesBeingWritten) {
743      return filesBeingWritten.isEmpty();
744    }
745  }
746  
747  /** @return true if the client is running */
748  boolean isClientRunning() {
749    return clientRunning;
750  }
751
752  long getLastLeaseRenewal() {
753    return lastLeaseRenewal;
754  }
755
756  void updateLastLeaseRenewal() {
757    synchronized(filesBeingWritten) {
758      if (filesBeingWritten.isEmpty()) {
759        return;
760      }
761      lastLeaseRenewal = Time.now();
762    }
763  }
764
765  /**
766   * Renew leases.
767   * @return true if lease was renewed. May return false if this
768   * client has been closed or has no files open.
769   **/
770  boolean renewLease() throws IOException {
771    if (clientRunning && !isFilesBeingWrittenEmpty()) {
772      try {
773        namenode.renewLease(clientName);
774        updateLastLeaseRenewal();
775        return true;
776      } catch (IOException e) {
777        // Abort if the lease has already expired. 
778        final long elapsed = Time.now() - getLastLeaseRenewal();
779        if (elapsed > HdfsConstants.LEASE_HARDLIMIT_PERIOD) {
780          LOG.warn("Failed to renew lease for " + clientName + " for "
781              + (elapsed/1000) + " seconds (>= hard-limit ="
782              + (HdfsConstants.LEASE_HARDLIMIT_PERIOD/1000) + " seconds.) "
783              + "Closing all files being written ...", e);
784          closeAllFilesBeingWritten(true);
785        } else {
786          // Let the lease renewer handle it and retry.
787          throw e;
788        }
789      }
790    }
791    return false;
792  }
793  
794  /**
795   * Close connections the Namenode.
796   */
797  void closeConnectionToNamenode() {
798    RPC.stopProxy(namenode);
799  }
800  
801  /** Abort and release resources held.  Ignore all errors. */
802  void abort() {
803    if (mmapManager != null) {
804      MMAP_MANAGER_FACTORY.unref(mmapManager);
805      mmapManager = null;
806    }
807    clientRunning = false;
808    closeAllFilesBeingWritten(true);
809    try {
810      // remove reference to this client and stop the renewer,
811      // if there is no more clients under the renewer.
812      getLeaseRenewer().closeClient(this);
813    } catch (IOException ioe) {
814       LOG.info("Exception occurred while aborting the client " + ioe);
815    }
816    closeConnectionToNamenode();
817  }
818
819  /** Close/abort all files being written. */
820  private void closeAllFilesBeingWritten(final boolean abort) {
821    for(;;) {
822      final String src;
823      final DFSOutputStream out;
824      synchronized(filesBeingWritten) {
825        if (filesBeingWritten.isEmpty()) {
826          return;
827        }
828        src = filesBeingWritten.keySet().iterator().next();
829        out = filesBeingWritten.remove(src);
830      }
831      if (out != null) {
832        try {
833          if (abort) {
834            out.abort();
835          } else {
836            out.close();
837          }
838        } catch(IOException ie) {
839          LOG.error("Failed to " + (abort? "abort": "close") + " file " + src,
840              ie);
841        }
842      }
843    }
844  }
845
846  /**
847   * Close the file system, abandoning all of the leases and files being
848   * created and close connections to the namenode.
849   */
850  @Override
851  public synchronized void close() throws IOException {
852    if (mmapManager != null) {
853      MMAP_MANAGER_FACTORY.unref(mmapManager);
854      mmapManager = null;
855    }
856    if(clientRunning) {
857      closeAllFilesBeingWritten(false);
858      clientRunning = false;
859      getLeaseRenewer().closeClient(this);
860      // close connections to the namenode
861      closeConnectionToNamenode();
862    }
863  }
864
865  /**
866   * Get the default block size for this cluster
867   * @return the default block size in bytes
868   */
869  public long getDefaultBlockSize() {
870    return dfsClientConf.defaultBlockSize;
871  }
872    
873  /**
874   * @see ClientProtocol#getPreferredBlockSize(String)
875   */
876  public long getBlockSize(String f) throws IOException {
877    try {
878      return namenode.getPreferredBlockSize(f);
879    } catch (IOException ie) {
880      LOG.warn("Problem getting block size", ie);
881      throw ie;
882    }
883  }
884
885  /**
886   * Get server default values for a number of configuration params.
887   * @see ClientProtocol#getServerDefaults()
888   */
889  public FsServerDefaults getServerDefaults() throws IOException {
890    long now = Time.now();
891    if (now - serverDefaultsLastUpdate > SERVER_DEFAULTS_VALIDITY_PERIOD) {
892      serverDefaults = namenode.getServerDefaults();
893      serverDefaultsLastUpdate = now;
894    }
895    return serverDefaults;
896  }
897  
898  /**
899   * Get a canonical token service name for this client's tokens.  Null should
900   * be returned if the client is not using tokens.
901   * @return the token service for the client
902   */
903  @InterfaceAudience.LimitedPrivate( { "HDFS" }) 
904  public String getCanonicalServiceName() {
905    return (dtService != null) ? dtService.toString() : null;
906  }
907  
908  /**
909   * @see ClientProtocol#getDelegationToken(Text)
910   */
911  public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
912      throws IOException {
913    assert dtService != null;
914    Token<DelegationTokenIdentifier> token =
915      namenode.getDelegationToken(renewer);
916
917    if (token != null) {
918      token.setService(this.dtService);
919      LOG.info("Created " + DelegationTokenIdentifier.stringifyToken(token));
920    } else {
921      LOG.info("Cannot get delegation token from " + renewer);
922    }
923    return token;
924
925  }
926
927  /**
928   * Renew a delegation token
929   * @param token the token to renew
930   * @return the new expiration time
931   * @throws InvalidToken
932   * @throws IOException
933   * @deprecated Use Token.renew instead.
934   */
935  @Deprecated
936  public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
937      throws InvalidToken, IOException {
938    LOG.info("Renewing " + DelegationTokenIdentifier.stringifyToken(token));
939    try {
940      return token.renew(conf);
941    } catch (InterruptedException ie) {                                       
942      throw new RuntimeException("caught interrupted", ie);
943    } catch (RemoteException re) {
944      throw re.unwrapRemoteException(InvalidToken.class,
945                                     AccessControlException.class);
946    }
947  }
948  
949  private static Map<String, Boolean> localAddrMap = Collections
950      .synchronizedMap(new HashMap<String, Boolean>());
951  
952  static boolean isLocalAddress(InetSocketAddress targetAddr) {
953    InetAddress addr = targetAddr.getAddress();
954    Boolean cached = localAddrMap.get(addr.getHostAddress());
955    if (cached != null) {
956      if (LOG.isTraceEnabled()) {
957        LOG.trace("Address " + targetAddr +
958                  (cached ? " is local" : " is not local"));
959      }
960      return cached;
961    }
962    
963    boolean local = NetUtils.isLocalAddress(addr);
964
965    if (LOG.isTraceEnabled()) {
966      LOG.trace("Address " + targetAddr +
967                (local ? " is local" : " is not local"));
968    }
969    localAddrMap.put(addr.getHostAddress(), local);
970    return local;
971  }
972  
973  /**
974   * Should the block access token be refetched on an exception
975   * 
976   * @param ex Exception received
977   * @param targetAddr Target datanode address from where exception was received
978   * @return true if block access token has expired or invalid and it should be
979   *         refetched
980   */
981  private static boolean tokenRefetchNeeded(IOException ex,
982      InetSocketAddress targetAddr) {
983    /*
984     * Get a new access token and retry. Retry is needed in 2 cases. 1) When
985     * both NN and DN re-started while DFSClient holding a cached access token.
986     * 2) In the case that NN fails to update its access key at pre-set interval
987     * (by a wide margin) and subsequently restarts. In this case, DN
988     * re-registers itself with NN and receives a new access key, but DN will
989     * delete the old access key from its memory since it's considered expired
990     * based on the estimated expiration date.
991     */
992    if (ex instanceof InvalidBlockTokenException || ex instanceof InvalidToken) {
993      LOG.info("Access token was invalid when connecting to " + targetAddr
994          + " : " + ex);
995      return true;
996    }
997    return false;
998  }
999  
1000  /**
1001   * Cancel a delegation token
1002   * @param token the token to cancel
1003   * @throws InvalidToken
1004   * @throws IOException
1005   * @deprecated Use Token.cancel instead.
1006   */
1007  @Deprecated
1008  public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
1009      throws InvalidToken, IOException {
1010    LOG.info("Cancelling " + DelegationTokenIdentifier.stringifyToken(token));
1011    try {
1012      token.cancel(conf);
1013     } catch (InterruptedException ie) {                                       
1014      throw new RuntimeException("caught interrupted", ie);
1015    } catch (RemoteException re) {
1016      throw re.unwrapRemoteException(InvalidToken.class,
1017                                     AccessControlException.class);
1018    }
1019  }
1020  
1021  @InterfaceAudience.Private
1022  public static class Renewer extends TokenRenewer {
1023    
1024    static {
1025      //Ensure that HDFS Configuration files are loaded before trying to use
1026      // the renewer.
1027      HdfsConfiguration.init();
1028    }
1029    
1030    @Override
1031    public boolean handleKind(Text kind) {
1032      return DelegationTokenIdentifier.HDFS_DELEGATION_KIND.equals(kind);
1033    }
1034
1035    @SuppressWarnings("unchecked")
1036    @Override
1037    public long renew(Token<?> token, Configuration conf) throws IOException {
1038      Token<DelegationTokenIdentifier> delToken = 
1039        (Token<DelegationTokenIdentifier>) token;
1040      ClientProtocol nn = getNNProxy(delToken, conf);
1041      try {
1042        return nn.renewDelegationToken(delToken);
1043      } catch (RemoteException re) {
1044        throw re.unwrapRemoteException(InvalidToken.class, 
1045                                       AccessControlException.class);
1046      }
1047    }
1048
1049    @SuppressWarnings("unchecked")
1050    @Override
1051    public void cancel(Token<?> token, Configuration conf) throws IOException {
1052      Token<DelegationTokenIdentifier> delToken = 
1053          (Token<DelegationTokenIdentifier>) token;
1054      LOG.info("Cancelling " + 
1055               DelegationTokenIdentifier.stringifyToken(delToken));
1056      ClientProtocol nn = getNNProxy(delToken, conf);
1057      try {
1058        nn.cancelDelegationToken(delToken);
1059      } catch (RemoteException re) {
1060        throw re.unwrapRemoteException(InvalidToken.class,
1061            AccessControlException.class);
1062      }
1063    }
1064    
1065    private static ClientProtocol getNNProxy(
1066        Token<DelegationTokenIdentifier> token, Configuration conf)
1067        throws IOException {
1068      URI uri = HAUtil.getServiceUriFromToken(token);
1069      if (HAUtil.isTokenForLogicalUri(token) &&
1070          !HAUtil.isLogicalUri(conf, uri)) {
1071        // If the token is for a logical nameservice, but the configuration
1072        // we have disagrees about that, we can't actually renew it.
1073        // This can be the case in MR, for example, if the RM doesn't
1074        // have all of the HA clusters configured in its configuration.
1075        throw new IOException("Unable to map logical nameservice URI '" +
1076            uri + "' to a NameNode. Local configuration does not have " +
1077            "a failover proxy provider configured.");
1078      }
1079      
1080      NameNodeProxies.ProxyAndInfo<ClientProtocol> info =
1081        NameNodeProxies.createProxy(conf, uri, ClientProtocol.class);
1082      assert info.getDelegationTokenService().equals(token.getService()) :
1083        "Returned service '" + info.getDelegationTokenService().toString() +
1084        "' doesn't match expected service '" +
1085        token.getService().toString() + "'";
1086        
1087      return info.getProxy();
1088    }
1089
1090    @Override
1091    public boolean isManaged(Token<?> token) throws IOException {
1092      return true;
1093    }
1094    
1095  }
1096
1097  /**
1098   * Report corrupt blocks that were discovered by the client.
1099   * @see ClientProtocol#reportBadBlocks(LocatedBlock[])
1100   */
1101  public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
1102    namenode.reportBadBlocks(blocks);
1103  }
1104  
1105  public short getDefaultReplication() {
1106    return dfsClientConf.defaultReplication;
1107  }
1108  
1109  public LocatedBlocks getLocatedBlocks(String src, long start)
1110      throws IOException {
1111    return getLocatedBlocks(src, start, dfsClientConf.prefetchSize);
1112  }
1113
1114  /*
1115   * This is just a wrapper around callGetBlockLocations, but non-static so that
1116   * we can stub it out for tests.
1117   */
1118  @VisibleForTesting
1119  public LocatedBlocks getLocatedBlocks(String src, long start, long length)
1120      throws IOException {
1121    return callGetBlockLocations(namenode, src, start, length);
1122  }
1123
1124  /**
1125   * @see ClientProtocol#getBlockLocations(String, long, long)
1126   */
1127  static LocatedBlocks callGetBlockLocations(ClientProtocol namenode,
1128      String src, long start, long length) 
1129      throws IOException {
1130    try {
1131      return namenode.getBlockLocations(src, start, length);
1132    } catch(RemoteException re) {
1133      throw re.unwrapRemoteException(AccessControlException.class,
1134                                     FileNotFoundException.class,
1135                                     UnresolvedPathException.class);
1136    }
1137  }
1138
1139  /**
1140   * Recover a file's lease
1141   * @param src a file's path
1142   * @return true if the file is already closed
1143   * @throws IOException
1144   */
1145  boolean recoverLease(String src) throws IOException {
1146    checkOpen();
1147
1148    try {
1149      return namenode.recoverLease(src, clientName);
1150    } catch (RemoteException re) {
1151      throw re.unwrapRemoteException(FileNotFoundException.class,
1152                                     AccessControlException.class,
1153                                     UnresolvedPathException.class);
1154    }
1155  }
1156
1157  /**
1158   * Get block location info about file
1159   * 
1160   * getBlockLocations() returns a list of hostnames that store 
1161   * data for a specific file region.  It returns a set of hostnames
1162   * for every block within the indicated region.
1163   *
1164   * This function is very useful when writing code that considers
1165   * data-placement when performing operations.  For example, the
1166   * MapReduce system tries to schedule tasks on the same machines
1167   * as the data-block the task processes. 
1168   */
1169  public BlockLocation[] getBlockLocations(String src, long start, 
1170    long length) throws IOException, UnresolvedLinkException {
1171    LocatedBlocks blocks = getLocatedBlocks(src, start, length);
1172    BlockLocation[] locations =  DFSUtil.locatedBlocks2Locations(blocks);
1173    HdfsBlockLocation[] hdfsLocations = new HdfsBlockLocation[locations.length];
1174    for (int i = 0; i < locations.length; i++) {
1175      hdfsLocations[i] = new HdfsBlockLocation(locations[i], blocks.get(i));
1176    }
1177    return hdfsLocations;
1178  }
1179  
1180  /**
1181   * Get block location information about a list of {@link HdfsBlockLocation}.
1182   * Used by {@link DistributedFileSystem#getFileBlockStorageLocations(List)} to
1183   * get {@link BlockStorageLocation}s for blocks returned by
1184   * {@link DistributedFileSystem#getFileBlockLocations(org.apache.hadoop.fs.FileStatus, long, long)}
1185   * .
1186   * 
1187   * This is done by making a round of RPCs to the associated datanodes, asking
1188   * the volume of each block replica. The returned array of
1189   * {@link BlockStorageLocation} expose this information as a
1190   * {@link VolumeId}.
1191   * 
1192   * @param blockLocations
1193   *          target blocks on which to query volume location information
1194   * @return volumeBlockLocations original block array augmented with additional
1195   *         volume location information for each replica.
1196   */
1197  public BlockStorageLocation[] getBlockStorageLocations(
1198      List<BlockLocation> blockLocations) throws IOException,
1199      UnsupportedOperationException, InvalidBlockTokenException {
1200    if (!getConf().getHdfsBlocksMetadataEnabled) {
1201      throw new UnsupportedOperationException("Datanode-side support for " +
1202          "getVolumeBlockLocations() must also be enabled in the client " +
1203          "configuration.");
1204    }
1205    // Downcast blockLocations and fetch out required LocatedBlock(s)
1206    List<LocatedBlock> blocks = new ArrayList<LocatedBlock>();
1207    for (BlockLocation loc : blockLocations) {
1208      if (!(loc instanceof HdfsBlockLocation)) {
1209        throw new ClassCastException("DFSClient#getVolumeBlockLocations " +
1210            "expected to be passed HdfsBlockLocations");
1211      }
1212      HdfsBlockLocation hdfsLoc = (HdfsBlockLocation) loc;
1213      blocks.add(hdfsLoc.getLocatedBlock());
1214    }
1215    
1216    // Re-group the LocatedBlocks to be grouped by datanodes, with the values
1217    // a list of the LocatedBlocks on the datanode.
1218    Map<DatanodeInfo, List<LocatedBlock>> datanodeBlocks = 
1219        new LinkedHashMap<DatanodeInfo, List<LocatedBlock>>();
1220    for (LocatedBlock b : blocks) {
1221      for (DatanodeInfo info : b.getLocations()) {
1222        if (!datanodeBlocks.containsKey(info)) {
1223          datanodeBlocks.put(info, new ArrayList<LocatedBlock>());
1224        }
1225        List<LocatedBlock> l = datanodeBlocks.get(info);
1226        l.add(b);
1227      }
1228    }
1229        
1230    // Make RPCs to the datanodes to get volume locations for its replicas
1231    List<HdfsBlocksMetadata> metadatas = BlockStorageLocationUtil
1232        .queryDatanodesForHdfsBlocksMetadata(conf, datanodeBlocks,
1233            getConf().getFileBlockStorageLocationsNumThreads,
1234            getConf().getFileBlockStorageLocationsTimeout,
1235            getConf().connectToDnViaHostname);
1236    
1237    // Regroup the returned VolumeId metadata to again be grouped by
1238    // LocatedBlock rather than by datanode
1239    Map<LocatedBlock, List<VolumeId>> blockVolumeIds = BlockStorageLocationUtil
1240        .associateVolumeIdsWithBlocks(blocks, datanodeBlocks, metadatas);
1241    
1242    // Combine original BlockLocations with new VolumeId information
1243    BlockStorageLocation[] volumeBlockLocations = BlockStorageLocationUtil
1244        .convertToVolumeBlockLocations(blocks, blockVolumeIds);
1245
1246    return volumeBlockLocations;
1247  }
1248  
1249  public DFSInputStream open(String src) 
1250      throws IOException, UnresolvedLinkException {
1251    return open(src, dfsClientConf.ioBufferSize, true, null);
1252  }
1253
1254  /**
1255   * Create an input stream that obtains a nodelist from the
1256   * namenode, and then reads from all the right places.  Creates
1257   * inner subclass of InputStream that does the right out-of-band
1258   * work.
1259   * @deprecated Use {@link #open(String, int, boolean)} instead.
1260   */
1261  @Deprecated
1262  public DFSInputStream open(String src, int buffersize, boolean verifyChecksum,
1263                             FileSystem.Statistics stats)
1264      throws IOException, UnresolvedLinkException {
1265    return open(src, buffersize, verifyChecksum);
1266  }
1267  
1268
1269  /**
1270   * Create an input stream that obtains a nodelist from the
1271   * namenode, and then reads from all the right places.  Creates
1272   * inner subclass of InputStream that does the right out-of-band
1273   * work.
1274   */
1275  public DFSInputStream open(String src, int buffersize, boolean verifyChecksum)
1276      throws IOException, UnresolvedLinkException {
1277    checkOpen();
1278    //    Get block info from namenode
1279    return new DFSInputStream(this, src, buffersize, verifyChecksum);
1280  }
1281
1282  /**
1283   * Get the namenode associated with this DFSClient object
1284   * @return the namenode associated with this DFSClient object
1285   */
1286  public ClientProtocol getNamenode() {
1287    return namenode;
1288  }
1289  
1290  /**
1291   * Call {@link #create(String, boolean, short, long, Progressable)} with
1292   * default <code>replication</code> and <code>blockSize<code> and null <code>
1293   * progress</code>.
1294   */
1295  public OutputStream create(String src, boolean overwrite) 
1296      throws IOException {
1297    return create(src, overwrite, dfsClientConf.defaultReplication,
1298        dfsClientConf.defaultBlockSize, null);
1299  }
1300    
1301  /**
1302   * Call {@link #create(String, boolean, short, long, Progressable)} with
1303   * default <code>replication</code> and <code>blockSize<code>.
1304   */
1305  public OutputStream create(String src, 
1306                             boolean overwrite,
1307                             Progressable progress) throws IOException {
1308    return create(src, overwrite, dfsClientConf.defaultReplication,
1309        dfsClientConf.defaultBlockSize, progress);
1310  }
1311    
1312  /**
1313   * Call {@link #create(String, boolean, short, long, Progressable)} with
1314   * null <code>progress</code>.
1315   */
1316  public OutputStream create(String src, 
1317                             boolean overwrite, 
1318                             short replication,
1319                             long blockSize) throws IOException {
1320    return create(src, overwrite, replication, blockSize, null);
1321  }
1322
1323  /**
1324   * Call {@link #create(String, boolean, short, long, Progressable, int)}
1325   * with default bufferSize.
1326   */
1327  public OutputStream create(String src, boolean overwrite, short replication,
1328      long blockSize, Progressable progress) throws IOException {
1329    return create(src, overwrite, replication, blockSize, progress,
1330        dfsClientConf.ioBufferSize);
1331  }
1332
1333  /**
1334   * Call {@link #create(String, FsPermission, EnumSet, short, long, 
1335   * Progressable, int, ChecksumOpt)} with default <code>permission</code>
1336   * {@link FsPermission#getFileDefault()}.
1337   * 
1338   * @param src File name
1339   * @param overwrite overwrite an existing file if true
1340   * @param replication replication factor for the file
1341   * @param blockSize maximum block size
1342   * @param progress interface for reporting client progress
1343   * @param buffersize underlying buffersize
1344   * 
1345   * @return output stream
1346   */
1347  public OutputStream create(String src,
1348                             boolean overwrite,
1349                             short replication,
1350                             long blockSize,
1351                             Progressable progress,
1352                             int buffersize)
1353      throws IOException {
1354    return create(src, FsPermission.getFileDefault(),
1355        overwrite ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
1356            : EnumSet.of(CreateFlag.CREATE), replication, blockSize, progress,
1357        buffersize, null);
1358  }
1359
1360  /**
1361   * Call {@link #create(String, FsPermission, EnumSet, boolean, short, 
1362   * long, Progressable, int, ChecksumOpt)} with <code>createParent</code>
1363   *  set to true.
1364   */
1365  public DFSOutputStream create(String src, 
1366                             FsPermission permission,
1367                             EnumSet<CreateFlag> flag, 
1368                             short replication,
1369                             long blockSize,
1370                             Progressable progress,
1371                             int buffersize,
1372                             ChecksumOpt checksumOpt)
1373      throws IOException {
1374    return create(src, permission, flag, true,
1375        replication, blockSize, progress, buffersize, checksumOpt, null);
1376  }
1377
1378  /**
1379   * Create a new dfs file with the specified block replication 
1380   * with write-progress reporting and return an output stream for writing
1381   * into the file.  
1382   * 
1383   * @param src File name
1384   * @param permission The permission of the directory being created.
1385   *          If null, use default permission {@link FsPermission#getFileDefault()}
1386   * @param flag indicates create a new file or create/overwrite an
1387   *          existing file or append to an existing file
1388   * @param createParent create missing parent directory if true
1389   * @param replication block replication
1390   * @param blockSize maximum block size
1391   * @param progress interface for reporting client progress
1392   * @param buffersize underlying buffer size 
1393   * @param checksumOpt checksum options
1394   * 
1395   * @return output stream
1396   * 
1397   * @see ClientProtocol#create(String, FsPermission, String, EnumSetWritable,
1398   * boolean, short, long) for detailed description of exceptions thrown
1399   */
1400  public DFSOutputStream create(String src, 
1401                             FsPermission permission,
1402                             EnumSet<CreateFlag> flag, 
1403                             boolean createParent,
1404                             short replication,
1405                             long blockSize,
1406                             Progressable progress,
1407                             int buffersize,
1408                             ChecksumOpt checksumOpt) throws IOException {
1409    return create(src, permission, flag, createParent, replication, blockSize, 
1410        progress, buffersize, checksumOpt, null);
1411  }
1412
1413  /**
1414   * Same as {@link #create(String, FsPermission, EnumSet, boolean, short, long,
1415   * Progressable, int, ChecksumOpt)} with the addition of favoredNodes that is
1416   * a hint to where the namenode should place the file blocks.
1417   * The favored nodes hint is not persisted in HDFS. Hence it may be honored
1418   * at the creation time only. HDFS could move the blocks during balancing or
1419   * replication, to move the blocks from favored nodes. A value of null means
1420   * no favored nodes for this create
1421   */
1422  public DFSOutputStream create(String src, 
1423                             FsPermission permission,
1424                             EnumSet<CreateFlag> flag, 
1425                             boolean createParent,
1426                             short replication,
1427                             long blockSize,
1428                             Progressable progress,
1429                             int buffersize,
1430                             ChecksumOpt checksumOpt,
1431                             InetSocketAddress[] favoredNodes) throws IOException {
1432    checkOpen();
1433    if (permission == null) {
1434      permission = FsPermission.getFileDefault();
1435    }
1436    FsPermission masked = permission.applyUMask(dfsClientConf.uMask);
1437    if(LOG.isDebugEnabled()) {
1438      LOG.debug(src + ": masked=" + masked);
1439    }
1440    String[] favoredNodeStrs = null;
1441    if (favoredNodes != null) {
1442      favoredNodeStrs = new String[favoredNodes.length];
1443      for (int i = 0; i < favoredNodes.length; i++) {
1444        favoredNodeStrs[i] = 
1445            favoredNodes[i].getHostName() + ":" 
1446                         + favoredNodes[i].getPort();
1447      }
1448    }
1449    final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this,
1450        src, masked, flag, createParent, replication, blockSize, progress,
1451        buffersize, dfsClientConf.createChecksum(checksumOpt), favoredNodeStrs);
1452    beginFileLease(src, result);
1453    return result;
1454  }
1455  
1456  /**
1457   * Append to an existing file if {@link CreateFlag#APPEND} is present
1458   */
1459  private DFSOutputStream primitiveAppend(String src, EnumSet<CreateFlag> flag,
1460      int buffersize, Progressable progress) throws IOException {
1461    if (flag.contains(CreateFlag.APPEND)) {
1462      HdfsFileStatus stat = getFileInfo(src);
1463      if (stat == null) { // No file to append to
1464        // New file needs to be created if create option is present
1465        if (!flag.contains(CreateFlag.CREATE)) {
1466          throw new FileNotFoundException("failed to append to non-existent file "
1467              + src + " on client " + clientName);
1468        }
1469        return null;
1470      }
1471      return callAppend(stat, src, buffersize, progress);
1472    }
1473    return null;
1474  }
1475  
1476  /**
1477   * Same as {{@link #create(String, FsPermission, EnumSet, short, long,
1478   *  Progressable, int, ChecksumOpt)} except that the permission
1479   *  is absolute (ie has already been masked with umask.
1480   */
1481  public DFSOutputStream primitiveCreate(String src, 
1482                             FsPermission absPermission,
1483                             EnumSet<CreateFlag> flag,
1484                             boolean createParent,
1485                             short replication,
1486                             long blockSize,
1487                             Progressable progress,
1488                             int buffersize,
1489                             ChecksumOpt checksumOpt)
1490      throws IOException, UnresolvedLinkException {
1491    checkOpen();
1492    CreateFlag.validate(flag);
1493    DFSOutputStream result = primitiveAppend(src, flag, buffersize, progress);
1494    if (result == null) {
1495      DataChecksum checksum = dfsClientConf.createChecksum(checksumOpt);
1496      result = DFSOutputStream.newStreamForCreate(this, src, absPermission,
1497          flag, createParent, replication, blockSize, progress, buffersize,
1498          checksum);
1499    }
1500    beginFileLease(src, result);
1501    return result;
1502  }
1503  
1504  /**
1505   * Creates a symbolic link.
1506   * 
1507   * @see ClientProtocol#createSymlink(String, String,FsPermission, boolean) 
1508   */
1509  public void createSymlink(String target, String link, boolean createParent)
1510      throws IOException {
1511    try {
1512      FsPermission dirPerm = 
1513          FsPermission.getDefault().applyUMask(dfsClientConf.uMask); 
1514      namenode.createSymlink(target, link, dirPerm, createParent);
1515    } catch (RemoteException re) {
1516      throw re.unwrapRemoteException(AccessControlException.class,
1517                                     FileAlreadyExistsException.class, 
1518                                     FileNotFoundException.class,
1519                                     ParentNotDirectoryException.class,
1520                                     NSQuotaExceededException.class, 
1521                                     DSQuotaExceededException.class,
1522                                     UnresolvedPathException.class,
1523                                     SnapshotAccessControlException.class);
1524    }
1525  }
1526
1527  /**
1528   * Resolve the *first* symlink, if any, in the path.
1529   * 
1530   * @see ClientProtocol#getLinkTarget(String)
1531   */
1532  public String getLinkTarget(String path) throws IOException { 
1533    checkOpen();
1534    try {
1535      return namenode.getLinkTarget(path);
1536    } catch (RemoteException re) {
1537      throw re.unwrapRemoteException(AccessControlException.class,
1538                                     FileNotFoundException.class);
1539    }
1540  }
1541
1542  /** Method to get stream returned by append call */
1543  private DFSOutputStream callAppend(HdfsFileStatus stat, String src,
1544      int buffersize, Progressable progress) throws IOException {
1545    LocatedBlock lastBlock = null;
1546    try {
1547      lastBlock = namenode.append(src, clientName);
1548    } catch(RemoteException re) {
1549      throw re.unwrapRemoteException(AccessControlException.class,
1550                                     FileNotFoundException.class,
1551                                     SafeModeException.class,
1552                                     DSQuotaExceededException.class,
1553                                     UnsupportedOperationException.class,
1554                                     UnresolvedPathException.class,
1555                                     SnapshotAccessControlException.class);
1556    }
1557    return DFSOutputStream.newStreamForAppend(this, src, buffersize, progress,
1558        lastBlock, stat, dfsClientConf.createChecksum());
1559  }
1560  
1561  /**
1562   * Append to an existing HDFS file.  
1563   * 
1564   * @param src file name
1565   * @param buffersize buffer size
1566   * @param progress for reporting write-progress; null is acceptable.
1567   * @param statistics file system statistics; null is acceptable.
1568   * @return an output stream for writing into the file
1569   * 
1570   * @see ClientProtocol#append(String, String) 
1571   */
1572  public HdfsDataOutputStream append(final String src, final int buffersize,
1573      final Progressable progress, final FileSystem.Statistics statistics
1574      ) throws IOException {
1575    final DFSOutputStream out = append(src, buffersize, progress);
1576    return new HdfsDataOutputStream(out, statistics, out.getInitialLen());
1577  }
1578
1579  private DFSOutputStream append(String src, int buffersize, Progressable progress) 
1580      throws IOException {
1581    checkOpen();
1582    HdfsFileStatus stat = getFileInfo(src);
1583    if (stat == null) { // No file found
1584      throw new FileNotFoundException("failed to append to non-existent file "
1585          + src + " on client " + clientName);
1586    }
1587    final DFSOutputStream result = callAppend(stat, src, buffersize, progress);
1588    beginFileLease(src, result);
1589    return result;
1590  }
1591
1592  /**
1593   * Set replication for an existing file.
1594   * @param src file name
1595   * @param replication
1596   * 
1597   * @see ClientProtocol#setReplication(String, short)
1598   */
1599  public boolean setReplication(String src, short replication)
1600      throws IOException {
1601    try {
1602      return namenode.setReplication(src, replication);
1603    } catch(RemoteException re) {
1604      throw re.unwrapRemoteException(AccessControlException.class,
1605                                     FileNotFoundException.class,
1606                                     SafeModeException.class,
1607                                     DSQuotaExceededException.class,
1608                                     UnresolvedPathException.class,
1609                                     SnapshotAccessControlException.class);
1610    }
1611  }
1612
1613  /**
1614   * Rename file or directory.
1615   * @see ClientProtocol#rename(String, String)
1616   * @deprecated Use {@link #rename(String, String, Options.Rename...)} instead.
1617   */
1618  @Deprecated
1619  public boolean rename(String src, String dst) throws IOException {
1620    checkOpen();
1621    try {
1622      return namenode.rename(src, dst);
1623    } catch(RemoteException re) {
1624      throw re.unwrapRemoteException(AccessControlException.class,
1625                                     NSQuotaExceededException.class,
1626                                     DSQuotaExceededException.class,
1627                                     UnresolvedPathException.class,
1628                                     SnapshotAccessControlException.class);
1629    }
1630  }
1631
1632  /**
1633   * Move blocks from src to trg and delete src
1634   * See {@link ClientProtocol#concat(String, String [])}. 
1635   */
1636  public void concat(String trg, String [] srcs) throws IOException {
1637    checkOpen();
1638    try {
1639      namenode.concat(trg, srcs);
1640    } catch(RemoteException re) {
1641      throw re.unwrapRemoteException(AccessControlException.class,
1642                                     UnresolvedPathException.class,
1643                                     SnapshotAccessControlException.class);
1644    }
1645  }
1646  /**
1647   * Rename file or directory.
1648   * @see ClientProtocol#rename2(String, String, Options.Rename...)
1649   */
1650  public void rename(String src, String dst, Options.Rename... options)
1651      throws IOException {
1652    checkOpen();
1653    try {
1654      namenode.rename2(src, dst, options);
1655    } catch(RemoteException re) {
1656      throw re.unwrapRemoteException(AccessControlException.class,
1657                                     DSQuotaExceededException.class,
1658                                     FileAlreadyExistsException.class,
1659                                     FileNotFoundException.class,
1660                                     ParentNotDirectoryException.class,
1661                                     SafeModeException.class,
1662                                     NSQuotaExceededException.class,
1663                                     UnresolvedPathException.class,
1664                                     SnapshotAccessControlException.class);
1665    }
1666  }
1667  /**
1668   * Delete file or directory.
1669   * See {@link ClientProtocol#delete(String, boolean)}. 
1670   */
1671  @Deprecated
1672  public boolean delete(String src) throws IOException {
1673    checkOpen();
1674    return namenode.delete(src, true);
1675  }
1676
1677  /**
1678   * delete file or directory.
1679   * delete contents of the directory if non empty and recursive 
1680   * set to true
1681   *
1682   * @see ClientProtocol#delete(String, boolean)
1683   */
1684  public boolean delete(String src, boolean recursive) throws IOException {
1685    checkOpen();
1686    try {
1687      return namenode.delete(src, recursive);
1688    } catch(RemoteException re) {
1689      throw re.unwrapRemoteException(AccessControlException.class,
1690                                     FileNotFoundException.class,
1691                                     SafeModeException.class,
1692                                     UnresolvedPathException.class,
1693                                     SnapshotAccessControlException.class);
1694    }
1695  }
1696  
1697  /** Implemented using getFileInfo(src)
1698   */
1699  public boolean exists(String src) throws IOException {
1700    checkOpen();
1701    return getFileInfo(src) != null;
1702  }
1703
1704  /**
1705   * Get a partial listing of the indicated directory
1706   * No block locations need to be fetched
1707   */
1708  public DirectoryListing listPaths(String src,  byte[] startAfter)
1709    throws IOException {
1710    return listPaths(src, startAfter, false);
1711  }
1712  
1713  /**
1714   * Get a partial listing of the indicated directory
1715   *
1716   * Recommend to use HdfsFileStatus.EMPTY_NAME as startAfter
1717   * if the application wants to fetch a listing starting from
1718   * the first entry in the directory
1719   *
1720   * @see ClientProtocol#getListing(String, byte[], boolean)
1721   */
1722  public DirectoryListing listPaths(String src,  byte[] startAfter,
1723      boolean needLocation) 
1724    throws IOException {
1725    checkOpen();
1726    try {
1727      return namenode.getListing(src, startAfter, needLocation);
1728    } catch(RemoteException re) {
1729      throw re.unwrapRemoteException(AccessControlException.class,
1730                                     FileNotFoundException.class,
1731                                     UnresolvedPathException.class);
1732    }
1733  }
1734
1735  /**
1736   * Get the file info for a specific file or directory.
1737   * @param src The string representation of the path to the file
1738   * @return object containing information regarding the file
1739   *         or null if file not found
1740   *         
1741   * @see ClientProtocol#getFileInfo(String) for description of exceptions
1742   */
1743  public HdfsFileStatus getFileInfo(String src) throws IOException {
1744    checkOpen();
1745    try {
1746      return namenode.getFileInfo(src);
1747    } catch(RemoteException re) {
1748      throw re.unwrapRemoteException(AccessControlException.class,
1749                                     FileNotFoundException.class,
1750                                     UnresolvedPathException.class);
1751    }
1752  }
1753  
1754  /**
1755   * Close status of a file
1756   * @return true if file is already closed
1757   */
1758  public boolean isFileClosed(String src) throws IOException{
1759    checkOpen();
1760    try {
1761      return namenode.isFileClosed(src);
1762    } catch(RemoteException re) {
1763      throw re.unwrapRemoteException(AccessControlException.class,
1764                                     FileNotFoundException.class,
1765                                     UnresolvedPathException.class);
1766    }
1767  }
1768  
1769  /**
1770   * Get the file info for a specific file or directory. If src
1771   * refers to a symlink then the FileStatus of the link is returned.
1772   * @param src path to a file or directory.
1773   * 
1774   * For description of exceptions thrown 
1775   * @see ClientProtocol#getFileLinkInfo(String)
1776   */
1777  public HdfsFileStatus getFileLinkInfo(String src) throws IOException {
1778    checkOpen();
1779    try {
1780      return namenode.getFileLinkInfo(src);
1781    } catch(RemoteException re) {
1782      throw re.unwrapRemoteException(AccessControlException.class,
1783                                     UnresolvedPathException.class);
1784     }
1785   }
1786
1787  /**
1788   * Get the checksum of a file.
1789   * @param src The file path
1790   * @return The checksum 
1791   * @see DistributedFileSystem#getFileChecksum(Path)
1792   */
1793  public MD5MD5CRC32FileChecksum getFileChecksum(String src) throws IOException {
1794    checkOpen();
1795    return getFileChecksum(src, clientName, namenode, socketFactory,
1796        dfsClientConf.socketTimeout, getDataEncryptionKey(),
1797        dfsClientConf.connectToDnViaHostname);
1798  }
1799  
1800  @InterfaceAudience.Private
1801  public void clearDataEncryptionKey() {
1802    LOG.debug("Clearing encryption key");
1803    synchronized (this) {
1804      encryptionKey = null;
1805    }
1806  }
1807  
1808  /**
1809   * @return true if data sent between this client and DNs should be encrypted,
1810   *         false otherwise.
1811   * @throws IOException in the event of error communicating with the NN
1812   */
1813  boolean shouldEncryptData() throws IOException {
1814    FsServerDefaults d = getServerDefaults();
1815    return d == null ? false : d.getEncryptDataTransfer();
1816  }
1817  
1818  @InterfaceAudience.Private
1819  public DataEncryptionKey getDataEncryptionKey()
1820      throws IOException {
1821    if (shouldEncryptData()) {
1822      synchronized (this) {
1823        if (encryptionKey == null ||
1824            encryptionKey.expiryDate < Time.now()) {
1825          LOG.debug("Getting new encryption token from NN");
1826          encryptionKey = namenode.getDataEncryptionKey();
1827        }
1828        return encryptionKey;
1829      }
1830    } else {
1831      return null;
1832    }
1833  }
1834
1835  /**
1836   * Get the checksum of a file.
1837   * @param src The file path
1838   * @param clientName the name of the client requesting the checksum.
1839   * @param namenode the RPC proxy for the namenode
1840   * @param socketFactory to create sockets to connect to DNs
1841   * @param socketTimeout timeout to use when connecting and waiting for a response
1842   * @param encryptionKey the key needed to communicate with DNs in this cluster
1843   * @param connectToDnViaHostname whether the client should use hostnames instead of IPs
1844   * @return The checksum 
1845   */
1846  private static MD5MD5CRC32FileChecksum getFileChecksum(String src,
1847      String clientName,
1848      ClientProtocol namenode, SocketFactory socketFactory, int socketTimeout,
1849      DataEncryptionKey encryptionKey, boolean connectToDnViaHostname)
1850      throws IOException {
1851    //get all block locations
1852    LocatedBlocks blockLocations = callGetBlockLocations(namenode, src, 0, Long.MAX_VALUE);
1853    if (null == blockLocations) {
1854      throw new FileNotFoundException("File does not exist: " + src);
1855    }
1856    List<LocatedBlock> locatedblocks = blockLocations.getLocatedBlocks();
1857    final DataOutputBuffer md5out = new DataOutputBuffer();
1858    int bytesPerCRC = -1;
1859    DataChecksum.Type crcType = DataChecksum.Type.DEFAULT;
1860    long crcPerBlock = 0;
1861    boolean refetchBlocks = false;
1862    int lastRetriedIndex = -1;
1863
1864    //get block checksum for each block
1865    for(int i = 0; i < locatedblocks.size(); i++) {
1866      if (refetchBlocks) {  // refetch to get fresh tokens
1867        blockLocations = callGetBlockLocations(namenode, src, 0, Long.MAX_VALUE);
1868        if (null == blockLocations) {
1869          throw new FileNotFoundException("File does not exist: " + src);
1870        }
1871        locatedblocks = blockLocations.getLocatedBlocks();
1872        refetchBlocks = false;
1873      }
1874      LocatedBlock lb = locatedblocks.get(i);
1875      final ExtendedBlock block = lb.getBlock();
1876      final DatanodeInfo[] datanodes = lb.getLocations();
1877      
1878      //try each datanode location of the block
1879      final int timeout = 3000 * datanodes.length + socketTimeout;
1880      boolean done = false;
1881      for(int j = 0; !done && j < datanodes.length; j++) {
1882        DataOutputStream out = null;
1883        DataInputStream in = null;
1884        
1885        try {
1886          //connect to a datanode
1887          IOStreamPair pair = connectToDN(socketFactory, connectToDnViaHostname,
1888              encryptionKey, datanodes[j], timeout);
1889          out = new DataOutputStream(new BufferedOutputStream(pair.out,
1890              HdfsConstants.SMALL_BUFFER_SIZE));
1891          in = new DataInputStream(pair.in);
1892
1893          if (LOG.isDebugEnabled()) {
1894            LOG.debug("write to " + datanodes[j] + ": "
1895                + Op.BLOCK_CHECKSUM + ", block=" + block);
1896          }
1897          // get block MD5
1898          new Sender(out).blockChecksum(block, lb.getBlockToken());
1899
1900          final BlockOpResponseProto reply =
1901            BlockOpResponseProto.parseFrom(PBHelper.vintPrefixed(in));
1902
1903          if (reply.getStatus() != Status.SUCCESS) {
1904            if (reply.getStatus() == Status.ERROR_ACCESS_TOKEN) {
1905              throw new InvalidBlockTokenException();
1906            } else {
1907              throw new IOException("Bad response " + reply + " for block "
1908                  + block + " from datanode " + datanodes[j]);
1909            }
1910          }
1911          
1912          OpBlockChecksumResponseProto checksumData =
1913            reply.getChecksumResponse();
1914
1915          //read byte-per-checksum
1916          final int bpc = checksumData.getBytesPerCrc();
1917          if (i == 0) { //first block
1918            bytesPerCRC = bpc;
1919          }
1920          else if (bpc != bytesPerCRC) {
1921            throw new IOException("Byte-per-checksum not matched: bpc=" + bpc
1922                + " but bytesPerCRC=" + bytesPerCRC);
1923          }
1924          
1925          //read crc-per-block
1926          final long cpb = checksumData.getCrcPerBlock();
1927          if (locatedblocks.size() > 1 && i == 0) {
1928            crcPerBlock = cpb;
1929          }
1930
1931          //read md5
1932          final MD5Hash md5 = new MD5Hash(
1933              checksumData.getMd5().toByteArray());
1934          md5.write(md5out);
1935          
1936          // read crc-type
1937          final DataChecksum.Type ct;
1938          if (checksumData.hasCrcType()) {
1939            ct = PBHelper.convert(checksumData
1940                .getCrcType());
1941          } else {
1942            LOG.debug("Retrieving checksum from an earlier-version DataNode: " +
1943                      "inferring checksum by reading first byte");
1944            ct = inferChecksumTypeByReading(
1945                clientName, socketFactory, socketTimeout, lb, datanodes[j],
1946                encryptionKey, connectToDnViaHostname);
1947          }
1948
1949          if (i == 0) { // first block
1950            crcType = ct;
1951          } else if (crcType != DataChecksum.Type.MIXED
1952              && crcType != ct) {
1953            // if crc types are mixed in a file
1954            crcType = DataChecksum.Type.MIXED;
1955          }
1956
1957          done = true;
1958
1959          if (LOG.isDebugEnabled()) {
1960            if (i == 0) {
1961              LOG.debug("set bytesPerCRC=" + bytesPerCRC
1962                  + ", crcPerBlock=" + crcPerBlock);
1963            }
1964            LOG.debug("got reply from " + datanodes[j] + ": md5=" + md5);
1965          }
1966        } catch (InvalidBlockTokenException ibte) {
1967          if (i > lastRetriedIndex) {
1968            if (LOG.isDebugEnabled()) {
1969              LOG.debug("Got access token error in response to OP_BLOCK_CHECKSUM "
1970                  + "for file " + src + " for block " + block
1971                  + " from datanode " + datanodes[j]
1972                  + ". Will retry the block once.");
1973            }
1974            lastRetriedIndex = i;
1975            done = true; // actually it's not done; but we'll retry
1976            i--; // repeat at i-th block
1977            refetchBlocks = true;
1978            break;
1979          }
1980        } catch (IOException ie) {
1981          LOG.warn("src=" + src + ", datanodes["+j+"]=" + datanodes[j], ie);
1982        } finally {
1983          IOUtils.closeStream(in);
1984          IOUtils.closeStream(out);
1985        }
1986      }
1987
1988      if (!done) {
1989        throw new IOException("Fail to get block MD5 for " + block);
1990      }
1991    }
1992
1993    //compute file MD5
1994    final MD5Hash fileMD5 = MD5Hash.digest(md5out.getData()); 
1995    switch (crcType) {
1996      case CRC32:
1997        return new MD5MD5CRC32GzipFileChecksum(bytesPerCRC,
1998            crcPerBlock, fileMD5);
1999      case CRC32C:
2000        return new MD5MD5CRC32CastagnoliFileChecksum(bytesPerCRC,
2001            crcPerBlock, fileMD5);
2002      default:
2003        // If there is no block allocated for the file,
2004        // return one with the magic entry that matches what previous
2005        // hdfs versions return.
2006        if (locatedblocks.size() == 0) {
2007          return new MD5MD5CRC32GzipFileChecksum(0, 0, fileMD5);
2008        }
2009
2010        // we should never get here since the validity was checked
2011        // when getCrcType() was called above.
2012        return null;
2013    }
2014  }
2015
2016  /**
2017   * Connect to the given datanode's datantrasfer port, and return
2018   * the resulting IOStreamPair. This includes encryption wrapping, etc.
2019   */
2020  private static IOStreamPair connectToDN(
2021      SocketFactory socketFactory, boolean connectToDnViaHostname,
2022      DataEncryptionKey encryptionKey, DatanodeInfo dn, int timeout)
2023      throws IOException
2024  {
2025    boolean success = false;
2026    Socket sock = null;
2027    try {
2028      sock = socketFactory.createSocket();
2029      String dnAddr = dn.getXferAddr(connectToDnViaHostname);
2030      if (LOG.isDebugEnabled()) {
2031        LOG.debug("Connecting to datanode " + dnAddr);
2032      }
2033      NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout);
2034      sock.setSoTimeout(timeout);
2035  
2036      OutputStream unbufOut = NetUtils.getOutputStream(sock);
2037      InputStream unbufIn = NetUtils.getInputStream(sock);
2038      IOStreamPair ret;
2039      if (encryptionKey != null) {
2040        ret = DataTransferEncryptor.getEncryptedStreams(
2041                unbufOut, unbufIn, encryptionKey);
2042      } else {
2043        ret = new IOStreamPair(unbufIn, unbufOut);        
2044      }
2045      success = true;
2046      return ret;
2047    } finally {
2048      if (!success) {
2049        IOUtils.closeSocket(sock);
2050      }
2051    }
2052  }
2053  
2054  /**
2055   * Infer the checksum type for a replica by sending an OP_READ_BLOCK
2056   * for the first byte of that replica. This is used for compatibility
2057   * with older HDFS versions which did not include the checksum type in
2058   * OpBlockChecksumResponseProto.
2059   *
2060   * @param in input stream from datanode
2061   * @param out output stream to datanode
2062   * @param lb the located block
2063   * @param clientName the name of the DFSClient requesting the checksum
2064   * @param dn the connected datanode
2065   * @return the inferred checksum type
2066   * @throws IOException if an error occurs
2067   */
2068  private static Type inferChecksumTypeByReading(
2069      String clientName, SocketFactory socketFactory, int socketTimeout,
2070      LocatedBlock lb, DatanodeInfo dn,
2071      DataEncryptionKey encryptionKey, boolean connectToDnViaHostname)
2072      throws IOException {
2073    IOStreamPair pair = connectToDN(socketFactory, connectToDnViaHostname,
2074        encryptionKey, dn, socketTimeout);
2075
2076    try {
2077      DataOutputStream out = new DataOutputStream(new BufferedOutputStream(pair.out,
2078          HdfsConstants.SMALL_BUFFER_SIZE));
2079      DataInputStream in = new DataInputStream(pair.in);
2080  
2081      new Sender(out).readBlock(lb.getBlock(), lb.getBlockToken(), clientName,
2082          0, 1, true, CachingStrategy.newDefaultStrategy());
2083      final BlockOpResponseProto reply =
2084          BlockOpResponseProto.parseFrom(PBHelper.vintPrefixed(in));
2085      
2086      if (reply.getStatus() != Status.SUCCESS) {
2087        if (reply.getStatus() == Status.ERROR_ACCESS_TOKEN) {
2088          throw new InvalidBlockTokenException();
2089        } else {
2090          throw new IOException("Bad response " + reply + " trying to read "
2091              + lb.getBlock() + " from datanode " + dn);
2092        }
2093      }
2094      
2095      return PBHelper.convert(reply.getReadOpChecksumInfo().getChecksum().getType());
2096    } finally {
2097      IOUtils.cleanup(null, pair.in, pair.out);
2098    }
2099  }
2100
2101  /**
2102   * Set permissions to a file or directory.
2103   * @param src path name.
2104   * @param permission
2105   * 
2106   * @see ClientProtocol#setPermission(String, FsPermission)
2107   */
2108  public void setPermission(String src, FsPermission permission)
2109      throws IOException {
2110    checkOpen();
2111    try {
2112      namenode.setPermission(src, permission);
2113    } catch(RemoteException re) {
2114      throw re.unwrapRemoteException(AccessControlException.class,
2115                                     FileNotFoundException.class,
2116                                     SafeModeException.class,
2117                                     UnresolvedPathException.class,
2118                                     SnapshotAccessControlException.class);
2119    }
2120  }
2121
2122  /**
2123   * Set file or directory owner.
2124   * @param src path name.
2125   * @param username user id.
2126   * @param groupname user group.
2127   * 
2128   * @see ClientProtocol#setOwner(String, String, String)
2129   */
2130  public void setOwner(String src, String username, String groupname)
2131      throws IOException {
2132    checkOpen();
2133    try {
2134      namenode.setOwner(src, username, groupname);
2135    } catch(RemoteException re) {
2136      throw re.unwrapRemoteException(AccessControlException.class,
2137                                     FileNotFoundException.class,
2138                                     SafeModeException.class,
2139                                     UnresolvedPathException.class,
2140                                     SnapshotAccessControlException.class);                                   
2141    }
2142  }
2143
2144  /**
2145   * @see ClientProtocol#getStats()
2146   */
2147  public FsStatus getDiskStatus() throws IOException {
2148    long rawNums[] = namenode.getStats();
2149    return new FsStatus(rawNums[0], rawNums[1], rawNums[2]);
2150  }
2151
2152  /**
2153   * Returns count of blocks with no good replicas left. Normally should be 
2154   * zero.
2155   * @throws IOException
2156   */ 
2157  public long getMissingBlocksCount() throws IOException {
2158    return namenode.getStats()[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX];
2159  }
2160  
2161  /**
2162   * Returns count of blocks with one of more replica missing.
2163   * @throws IOException
2164   */ 
2165  public long getUnderReplicatedBlocksCount() throws IOException {
2166    return namenode.getStats()[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX];
2167  }
2168  
2169  /**
2170   * Returns count of blocks with at least one replica marked corrupt. 
2171   * @throws IOException
2172   */ 
2173  public long getCorruptBlocksCount() throws IOException {
2174    return namenode.getStats()[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX];
2175  }
2176  
2177  /**
2178   * @return a list in which each entry describes a corrupt file/block
2179   * @throws IOException
2180   */
2181  public CorruptFileBlocks listCorruptFileBlocks(String path,
2182                                                 String cookie)
2183    throws IOException {
2184    return namenode.listCorruptFileBlocks(path, cookie);
2185  }
2186
2187  public DatanodeInfo[] datanodeReport(DatanodeReportType type)
2188  throws IOException {
2189    return namenode.getDatanodeReport(type);
2190  }
2191    
2192  /**
2193   * Enter, leave or get safe mode.
2194   * 
2195   * @see ClientProtocol#setSafeMode(HdfsConstants.SafeModeAction,boolean)
2196   */
2197  public boolean setSafeMode(SafeModeAction action) throws IOException {
2198    return setSafeMode(action, false);
2199  }
2200  
2201  /**
2202   * Enter, leave or get safe mode.
2203   * 
2204   * @param action
2205   *          One of SafeModeAction.GET, SafeModeAction.ENTER and
2206   *          SafeModeActiob.LEAVE
2207   * @param isChecked
2208   *          If true, then check only active namenode's safemode status, else
2209   *          check first namenode's status.
2210   * @see ClientProtocol#setSafeMode(HdfsConstants.SafeModeAction, boolean)
2211   */
2212  public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException{
2213    return namenode.setSafeMode(action, isChecked);    
2214  }
2215 
2216  /**
2217   * Create one snapshot.
2218   * 
2219   * @param snapshotRoot The directory where the snapshot is to be taken
2220   * @param snapshotName Name of the snapshot
2221   * @return the snapshot path.
2222   * @see ClientProtocol#createSnapshot(String, String)
2223   */
2224  public String createSnapshot(String snapshotRoot, String snapshotName)
2225      throws IOException {
2226    checkOpen();
2227    try {
2228      return namenode.createSnapshot(snapshotRoot, snapshotName);
2229    } catch(RemoteException re) {
2230      throw re.unwrapRemoteException();
2231    }
2232  }
2233  
2234  /**
2235   * Delete a snapshot of a snapshottable directory.
2236   * 
2237   * @param snapshotRoot The snapshottable directory that the 
2238   *                    to-be-deleted snapshot belongs to
2239   * @param snapshotName The name of the to-be-deleted snapshot
2240   * @throws IOException
2241   * @see ClientProtocol#deleteSnapshot(String, String)
2242   */
2243  public void deleteSnapshot(String snapshotRoot, String snapshotName)
2244      throws IOException {
2245    try {
2246      namenode.deleteSnapshot(snapshotRoot, snapshotName);
2247    } catch(RemoteException re) {
2248      throw re.unwrapRemoteException();
2249    }
2250  }
2251  
2252  /**
2253   * Rename a snapshot.
2254   * @param snapshotDir The directory path where the snapshot was taken
2255   * @param snapshotOldName Old name of the snapshot
2256   * @param snapshotNewName New name of the snapshot
2257   * @throws IOException
2258   * @see ClientProtocol#renameSnapshot(String, String, String)
2259   */
2260  public void renameSnapshot(String snapshotDir, String snapshotOldName,
2261      String snapshotNewName) throws IOException {
2262    checkOpen();
2263    try {
2264      namenode.renameSnapshot(snapshotDir, snapshotOldName, snapshotNewName);
2265    } catch(RemoteException re) {
2266      throw re.unwrapRemoteException();
2267    }
2268  }
2269  
2270  /**
2271   * Get all the current snapshottable directories.
2272   * @return All the current snapshottable directories
2273   * @throws IOException
2274   * @see ClientProtocol#getSnapshottableDirListing()
2275   */
2276  public SnapshottableDirectoryStatus[] getSnapshottableDirListing()
2277      throws IOException {
2278    checkOpen();
2279    try {
2280      return namenode.getSnapshottableDirListing();
2281    } catch(RemoteException re) {
2282      throw re.unwrapRemoteException();
2283    }
2284  }
2285
2286  /**
2287   * Allow snapshot on a directory.
2288   * 
2289   * @see ClientProtocol#allowSnapshot(String snapshotRoot)
2290   */
2291  public void allowSnapshot(String snapshotRoot) throws IOException {
2292    checkOpen();
2293    try {
2294      namenode.allowSnapshot(snapshotRoot);
2295    } catch (RemoteException re) {
2296      throw re.unwrapRemoteException();
2297    }
2298  }
2299  
2300  /**
2301   * Disallow snapshot on a directory.
2302   * 
2303   * @see ClientProtocol#disallowSnapshot(String snapshotRoot)
2304   */
2305  public void disallowSnapshot(String snapshotRoot) throws IOException {
2306    checkOpen();
2307    try {
2308      namenode.disallowSnapshot(snapshotRoot);
2309    } catch (RemoteException re) {
2310      throw re.unwrapRemoteException();
2311    }
2312  }
2313  
2314  /**
2315   * Get the difference between two snapshots, or between a snapshot and the
2316   * current tree of a directory.
2317   * @see ClientProtocol#getSnapshotDiffReport(String, String, String)
2318   */
2319  public SnapshotDiffReport getSnapshotDiffReport(String snapshotDir,
2320      String fromSnapshot, String toSnapshot) throws IOException {
2321    checkOpen();
2322    try {
2323      return namenode.getSnapshotDiffReport(snapshotDir,
2324          fromSnapshot, toSnapshot);
2325    } catch(RemoteException re) {
2326      throw re.unwrapRemoteException();
2327    }
2328  }
2329
2330  public long addCacheDirective(
2331      CacheDirectiveInfo info, EnumSet<CacheFlag> flags) throws IOException {
2332    checkOpen();
2333    try {
2334      return namenode.addCacheDirective(info, flags);
2335    } catch (RemoteException re) {
2336      throw re.unwrapRemoteException();
2337    }
2338  }
2339  
2340  public void modifyCacheDirective(
2341      CacheDirectiveInfo info, EnumSet<CacheFlag> flags) throws IOException {
2342    checkOpen();
2343    try {
2344      namenode.modifyCacheDirective(info, flags);
2345    } catch (RemoteException re) {
2346      throw re.unwrapRemoteException();
2347    }
2348  }
2349
2350  public void removeCacheDirective(long id)
2351      throws IOException {
2352    checkOpen();
2353    try {
2354      namenode.removeCacheDirective(id);
2355    } catch (RemoteException re) {
2356      throw re.unwrapRemoteException();
2357    }
2358  }
2359  
2360  public RemoteIterator<CacheDirectiveEntry> listCacheDirectives(
2361      CacheDirectiveInfo filter) throws IOException {
2362    return new CacheDirectiveIterator(namenode, filter);
2363  }
2364
2365  public void addCachePool(CachePoolInfo info) throws IOException {
2366    checkOpen();
2367    try {
2368      namenode.addCachePool(info);
2369    } catch (RemoteException re) {
2370      throw re.unwrapRemoteException();
2371    }
2372  }
2373
2374  public void modifyCachePool(CachePoolInfo info) throws IOException {
2375    checkOpen();
2376    try {
2377      namenode.modifyCachePool(info);
2378    } catch (RemoteException re) {
2379      throw re.unwrapRemoteException();
2380    }
2381  }
2382
2383  public void removeCachePool(String poolName) throws IOException {
2384    checkOpen();
2385    try {
2386      namenode.removeCachePool(poolName);
2387    } catch (RemoteException re) {
2388      throw re.unwrapRemoteException();
2389    }
2390  }
2391
2392  public RemoteIterator<CachePoolEntry> listCachePools() throws IOException {
2393    return new CachePoolIterator(namenode);
2394  }
2395
2396  /**
2397   * Save namespace image.
2398   * 
2399   * @see ClientProtocol#saveNamespace()
2400   */
2401  void saveNamespace() throws AccessControlException, IOException {
2402    try {
2403      namenode.saveNamespace();
2404    } catch(RemoteException re) {
2405      throw re.unwrapRemoteException(AccessControlException.class);
2406    }
2407  }
2408
2409  /**
2410   * Rolls the edit log on the active NameNode.
2411   * @return the txid of the new log segment 
2412   *
2413   * @see ClientProtocol#rollEdits()
2414   */
2415  long rollEdits() throws AccessControlException, IOException {
2416    try {
2417      return namenode.rollEdits();
2418    } catch(RemoteException re) {
2419      throw re.unwrapRemoteException(AccessControlException.class);
2420    }
2421  }
2422
2423  @VisibleForTesting
2424  ExtendedBlock getPreviousBlock(String file) {
2425    return filesBeingWritten.get(file).getBlock();
2426  }
2427  
2428  /**
2429   * enable/disable restore failed storage.
2430   * 
2431   * @see ClientProtocol#restoreFailedStorage(String arg)
2432   */
2433  boolean restoreFailedStorage(String arg)
2434      throws AccessControlException, IOException{
2435    return namenode.restoreFailedStorage(arg);
2436  }
2437
2438  /**
2439   * Refresh the hosts and exclude files.  (Rereads them.)
2440   * See {@link ClientProtocol#refreshNodes()} 
2441   * for more details.
2442   * 
2443   * @see ClientProtocol#refreshNodes()
2444   */
2445  public void refreshNodes() throws IOException {
2446    namenode.refreshNodes();
2447  }
2448
2449  /**
2450   * Dumps DFS data structures into specified file.
2451   * 
2452   * @see ClientProtocol#metaSave(String)
2453   */
2454  public void metaSave(String pathname) throws IOException {
2455    namenode.metaSave(pathname);
2456  }
2457
2458  /**
2459   * Requests the namenode to tell all datanodes to use a new, non-persistent
2460   * bandwidth value for dfs.balance.bandwidthPerSec.
2461   * See {@link ClientProtocol#setBalancerBandwidth(long)} 
2462   * for more details.
2463   * 
2464   * @see ClientProtocol#setBalancerBandwidth(long)
2465   */
2466  public void setBalancerBandwidth(long bandwidth) throws IOException {
2467    namenode.setBalancerBandwidth(bandwidth);
2468  }
2469    
2470  /**
2471   * @see ClientProtocol#finalizeUpgrade()
2472   */
2473  public void finalizeUpgrade() throws IOException {
2474    namenode.finalizeUpgrade();
2475  }
2476
2477  /**
2478   */
2479  @Deprecated
2480  public boolean mkdirs(String src) throws IOException {
2481    return mkdirs(src, null, true);
2482  }
2483
2484  /**
2485   * Create a directory (or hierarchy of directories) with the given
2486   * name and permission.
2487   *
2488   * @param src The path of the directory being created
2489   * @param permission The permission of the directory being created.
2490   * If permission == null, use {@link FsPermission#getDefault()}.
2491   * @param createParent create missing parent directory if true
2492   * 
2493   * @return True if the operation success.
2494   * 
2495   * @see ClientProtocol#mkdirs(String, FsPermission, boolean)
2496   */
2497  public boolean mkdirs(String src, FsPermission permission,
2498      boolean createParent) throws IOException {
2499    if (permission == null) {
2500      permission = FsPermission.getDefault();
2501    }
2502    FsPermission masked = permission.applyUMask(dfsClientConf.uMask);
2503    return primitiveMkdir(src, masked, createParent);
2504  }
2505
2506  /**
2507   * Same {{@link #mkdirs(String, FsPermission, boolean)} except
2508   * that the permissions has already been masked against umask.
2509   */
2510  public boolean primitiveMkdir(String src, FsPermission absPermission)
2511    throws IOException {
2512    return primitiveMkdir(src, absPermission, true);
2513  }
2514
2515  /**
2516   * Same {{@link #mkdirs(String, FsPermission, boolean)} except
2517   * that the permissions has already been masked against umask.
2518   */
2519  public boolean primitiveMkdir(String src, FsPermission absPermission, 
2520    boolean createParent)
2521    throws IOException {
2522    checkOpen();
2523    if (absPermission == null) {
2524      absPermission = 
2525        FsPermission.getDefault().applyUMask(dfsClientConf.uMask);
2526    } 
2527
2528    if(LOG.isDebugEnabled()) {
2529      LOG.debug(src + ": masked=" + absPermission);
2530    }
2531    try {
2532      return namenode.mkdirs(src, absPermission, createParent);
2533    } catch(RemoteException re) {
2534      throw re.unwrapRemoteException(AccessControlException.class,
2535                                     InvalidPathException.class,
2536                                     FileAlreadyExistsException.class,
2537                                     FileNotFoundException.class,
2538                                     ParentNotDirectoryException.class,
2539                                     SafeModeException.class,
2540                                     NSQuotaExceededException.class,
2541                                     DSQuotaExceededException.class,
2542                                     UnresolvedPathException.class,
2543                                     SnapshotAccessControlException.class);
2544    }
2545  }
2546  
2547  /**
2548   * Get {@link ContentSummary} rooted at the specified directory.
2549   * @param path The string representation of the path
2550   * 
2551   * @see ClientProtocol#getContentSummary(String)
2552   */
2553  ContentSummary getContentSummary(String src) throws IOException {
2554    try {
2555      return namenode.getContentSummary(src);
2556    } catch(RemoteException re) {
2557      throw re.unwrapRemoteException(AccessControlException.class,
2558                                     FileNotFoundException.class,
2559                                     UnresolvedPathException.class);
2560    }
2561  }
2562
2563  /**
2564   * Sets or resets quotas for a directory.
2565   * @see ClientProtocol#setQuota(String, long, long)
2566   */
2567  void setQuota(String src, long namespaceQuota, long diskspaceQuota) 
2568      throws IOException {
2569    // sanity check
2570    if ((namespaceQuota <= 0 && namespaceQuota != HdfsConstants.QUOTA_DONT_SET &&
2571         namespaceQuota != HdfsConstants.QUOTA_RESET) ||
2572        (diskspaceQuota <= 0 && diskspaceQuota != HdfsConstants.QUOTA_DONT_SET &&
2573         diskspaceQuota != HdfsConstants.QUOTA_RESET)) {
2574      throw new IllegalArgumentException("Invalid values for quota : " +
2575                                         namespaceQuota + " and " + 
2576                                         diskspaceQuota);
2577                                         
2578    }
2579    try {
2580      namenode.setQuota(src, namespaceQuota, diskspaceQuota);
2581    } catch(RemoteException re) {
2582      throw re.unwrapRemoteException(AccessControlException.class,
2583                                     FileNotFoundException.class,
2584                                     NSQuotaExceededException.class,
2585                                     DSQuotaExceededException.class,
2586                                     UnresolvedPathException.class,
2587                                     SnapshotAccessControlException.class);
2588    }
2589  }
2590
2591  /**
2592   * set the modification and access time of a file
2593   * 
2594   * @see ClientProtocol#setTimes(String, long, long)
2595   */
2596  public void setTimes(String src, long mtime, long atime) throws IOException {
2597    checkOpen();
2598    try {
2599      namenode.setTimes(src, mtime, atime);
2600    } catch(RemoteException re) {
2601      throw re.unwrapRemoteException(AccessControlException.class,
2602                                     FileNotFoundException.class,
2603                                     UnresolvedPathException.class,
2604                                     SnapshotAccessControlException.class);
2605    }
2606  }
2607
2608  /**
2609   * @deprecated use {@link HdfsDataInputStream} instead.
2610   */
2611  @Deprecated
2612  public static class DFSDataInputStream extends HdfsDataInputStream {
2613
2614    public DFSDataInputStream(DFSInputStream in) throws IOException {
2615      super(in);
2616    }
2617  }
2618
2619  void reportChecksumFailure(String file, ExtendedBlock blk, DatanodeInfo dn) {
2620    DatanodeInfo [] dnArr = { dn };
2621    LocatedBlock [] lblocks = { new LocatedBlock(blk, dnArr) };
2622    reportChecksumFailure(file, lblocks);
2623  }
2624    
2625  // just reports checksum failure and ignores any exception during the report.
2626  void reportChecksumFailure(String file, LocatedBlock lblocks[]) {
2627    try {
2628      reportBadBlocks(lblocks);
2629    } catch (IOException ie) {
2630      LOG.info("Found corruption while reading " + file
2631          + ". Error repairing corrupt blocks. Bad blocks remain.", ie);
2632    }
2633  }
2634
2635  @Override
2636  public String toString() {
2637    return getClass().getSimpleName() + "[clientName=" + clientName
2638        + ", ugi=" + ugi + "]"; 
2639  }
2640
2641  public DomainSocketFactory getDomainSocketFactory() {
2642    return domainSocketFactory;
2643  }
2644
2645  public void disableLegacyBlockReaderLocal() {
2646    shouldUseLegacyBlockReaderLocal = false;
2647  }
2648
2649  public boolean useLegacyBlockReaderLocal() {
2650    return shouldUseLegacyBlockReaderLocal;
2651  }
2652
2653  public CachingStrategy getDefaultReadCachingStrategy() {
2654    return defaultReadCachingStrategy;
2655  }
2656
2657  public CachingStrategy getDefaultWriteCachingStrategy() {
2658    return defaultWriteCachingStrategy;
2659  }
2660
2661  @VisibleForTesting
2662  public ClientMmapManager getMmapManager() {
2663    return mmapManager;
2664  }
2665}