001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs;
019
020import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT;
021import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY;
022import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX;
023import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT;
024import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY;
025import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT;
026import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY;
027import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY;
028import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT;
029
030import java.io.IOException;
031import java.lang.reflect.Constructor;
032import java.lang.reflect.InvocationHandler;
033import java.lang.reflect.Proxy;
034import java.net.InetSocketAddress;
035import java.net.URI;
036import java.util.HashMap;
037import java.util.Map;
038import java.util.concurrent.TimeUnit;
039
040import org.apache.commons.logging.Log;
041import org.apache.commons.logging.LogFactory;
042import org.apache.hadoop.conf.Configuration;
043import org.apache.hadoop.hdfs.DFSClient.Conf;
044import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
045import org.apache.hadoop.hdfs.protocol.ClientProtocol;
046import org.apache.hadoop.hdfs.protocol.HdfsConstants;
047import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolPB;
048import org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB;
049import org.apache.hadoop.hdfs.protocolPB.JournalProtocolPB;
050import org.apache.hadoop.hdfs.protocolPB.JournalProtocolTranslatorPB;
051import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolPB;
052import org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB;
053import org.apache.hadoop.hdfs.server.namenode.NameNode;
054import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
055import org.apache.hadoop.hdfs.server.protocol.JournalProtocol;
056import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
057import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
058import org.apache.hadoop.io.Text;
059import org.apache.hadoop.io.retry.DefaultFailoverProxyProvider;
060import org.apache.hadoop.io.retry.FailoverProxyProvider;
061import org.apache.hadoop.io.retry.LossyRetryInvocationHandler;
062import org.apache.hadoop.io.retry.RetryPolicies;
063import org.apache.hadoop.io.retry.RetryPolicy;
064import org.apache.hadoop.io.retry.RetryProxy;
065import org.apache.hadoop.io.retry.RetryUtils;
066import org.apache.hadoop.ipc.ProtobufRpcEngine;
067import org.apache.hadoop.ipc.RPC;
068import org.apache.hadoop.ipc.RemoteException;
069import org.apache.hadoop.net.NetUtils;
070import org.apache.hadoop.security.RefreshUserMappingsProtocol;
071import org.apache.hadoop.security.SecurityUtil;
072import org.apache.hadoop.security.UserGroupInformation;
073import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
074import org.apache.hadoop.security.protocolPB.RefreshAuthorizationPolicyProtocolClientSideTranslatorPB;
075import org.apache.hadoop.security.protocolPB.RefreshAuthorizationPolicyProtocolPB;
076import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolClientSideTranslatorPB;
077import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolPB;
078import org.apache.hadoop.tools.GetUserMappingsProtocol;
079import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolClientSideTranslatorPB;
080import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolPB;
081
082import com.google.common.annotations.VisibleForTesting;
083import com.google.common.base.Preconditions;
084
085/**
086 * Create proxy objects to communicate with a remote NN. All remote access to an
087 * NN should be funneled through this class. Most of the time you'll want to use
088 * {@link NameNodeProxies#createProxy(Configuration, URI, Class)}, which will
089 * create either an HA- or non-HA-enabled client proxy as appropriate.
090 */
091public class NameNodeProxies {
092  
093  private static final Log LOG = LogFactory.getLog(NameNodeProxies.class);
094
095  /**
096   * Wrapper for a client proxy as well as its associated service ID.
097   * This is simply used as a tuple-like return type for
098   * {@link NameNodeProxies#createProxy} and
099   * {@link NameNodeProxies#createNonHAProxy}.
100   */
101  public static class ProxyAndInfo<PROXYTYPE> {
102    private final PROXYTYPE proxy;
103    private final Text dtService;
104    
105    public ProxyAndInfo(PROXYTYPE proxy, Text dtService) {
106      this.proxy = proxy;
107      this.dtService = dtService;
108    }
109    
110    public PROXYTYPE getProxy() {
111      return proxy;
112    }
113    
114    public Text getDelegationTokenService() {
115      return dtService;
116    }
117  }
118
119  /**
120   * Creates the namenode proxy with the passed protocol. This will handle
121   * creation of either HA- or non-HA-enabled proxy objects, depending upon
122   * if the provided URI is a configured logical URI.
123   * 
124   * @param conf the configuration containing the required IPC
125   *        properties, client failover configurations, etc.
126   * @param nameNodeUri the URI pointing either to a specific NameNode
127   *        or to a logical nameservice.
128   * @param xface the IPC interface which should be created
129   * @return an object containing both the proxy and the associated
130   *         delegation token service it corresponds to
131   * @throws IOException if there is an error creating the proxy
132   **/
133  @SuppressWarnings("unchecked")
134  public static <T> ProxyAndInfo<T> createProxy(Configuration conf,
135      URI nameNodeUri, Class<T> xface) throws IOException {
136    Class<FailoverProxyProvider<T>> failoverProxyProviderClass =
137        getFailoverProxyProviderClass(conf, nameNodeUri, xface);
138  
139    if (failoverProxyProviderClass == null) {
140      // Non-HA case
141      return createNonHAProxy(conf, NameNode.getAddress(nameNodeUri), xface,
142          UserGroupInformation.getCurrentUser(), true);
143    } else {
144      // HA case
145      FailoverProxyProvider<T> failoverProxyProvider = NameNodeProxies
146          .createFailoverProxyProvider(conf, failoverProxyProviderClass, xface,
147              nameNodeUri);
148      Conf config = new Conf(conf);
149      T proxy = (T) RetryProxy.create(xface, failoverProxyProvider,
150          RetryPolicies.failoverOnNetworkException(
151              RetryPolicies.TRY_ONCE_THEN_FAIL, config.maxFailoverAttempts,
152              config.maxRetryAttempts, config.failoverSleepBaseMillis,
153              config.failoverSleepMaxMillis));
154      
155      Text dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri);
156      return new ProxyAndInfo<T>(proxy, dtService);
157    }
158  }
159  
160  /**
161   * Generate a dummy namenode proxy instance that utilizes our hacked
162   * {@link LossyRetryInvocationHandler}. Proxy instance generated using this
163   * method will proactively drop RPC responses. Currently this method only
164   * support HA setup. null will be returned if the given configuration is not 
165   * for HA.
166   * 
167   * @param config the configuration containing the required IPC
168   *        properties, client failover configurations, etc.
169   * @param nameNodeUri the URI pointing either to a specific NameNode
170   *        or to a logical nameservice.
171   * @param xface the IPC interface which should be created
172   * @param numResponseToDrop The number of responses to drop for each RPC call
173   * @return an object containing both the proxy and the associated
174   *         delegation token service it corresponds to. Will return null of the
175   *         given configuration does not support HA.
176   * @throws IOException if there is an error creating the proxy
177   */
178  @SuppressWarnings("unchecked")
179  public static <T> ProxyAndInfo<T> createProxyWithLossyRetryHandler(
180      Configuration config, URI nameNodeUri, Class<T> xface,
181      int numResponseToDrop) throws IOException {
182    Preconditions.checkArgument(numResponseToDrop > 0);
183    Class<FailoverProxyProvider<T>> failoverProxyProviderClass = 
184        getFailoverProxyProviderClass(config, nameNodeUri, xface);
185    if (failoverProxyProviderClass != null) { // HA case
186      FailoverProxyProvider<T> failoverProxyProvider = 
187          createFailoverProxyProvider(config, failoverProxyProviderClass, 
188              xface, nameNodeUri);
189      int delay = config.getInt(
190          DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY,
191          DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT);
192      int maxCap = config.getInt(
193          DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY,
194          DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT);
195      int maxFailoverAttempts = config.getInt(
196          DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY,
197          DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT);
198      int maxRetryAttempts = config.getInt(
199          DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY,
200          DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT);
201      InvocationHandler dummyHandler = new LossyRetryInvocationHandler<T>(
202              numResponseToDrop, failoverProxyProvider,
203              RetryPolicies.failoverOnNetworkException(
204                  RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts, 
205                  Math.max(numResponseToDrop + 1, maxRetryAttempts), delay, 
206                  maxCap));
207      
208      T proxy = (T) Proxy.newProxyInstance(
209          failoverProxyProvider.getInterface().getClassLoader(),
210          new Class[] { xface }, dummyHandler);
211      Text dtService = HAUtil.buildTokenServiceForLogicalUri(nameNodeUri);
212      return new ProxyAndInfo<T>(proxy, dtService);
213    } else {
214      LOG.warn("Currently creating proxy using " +
215                "LossyRetryInvocationHandler requires NN HA setup");
216      return null;
217    }
218  }
219
220  /**
221   * Creates an explicitly non-HA-enabled proxy object. Most of the time you
222   * don't want to use this, and should instead use {@link NameNodeProxies#createProxy}.
223   * 
224   * @param conf the configuration object
225   * @param nnAddr address of the remote NN to connect to
226   * @param xface the IPC interface which should be created
227   * @param ugi the user who is making the calls on the proxy object
228   * @param withRetries certain interfaces have a non-standard retry policy
229   * @return an object containing both the proxy and the associated
230   *         delegation token service it corresponds to
231   * @throws IOException
232   */
233  @SuppressWarnings("unchecked")
234  public static <T> ProxyAndInfo<T> createNonHAProxy(
235      Configuration conf, InetSocketAddress nnAddr, Class<T> xface,
236      UserGroupInformation ugi, boolean withRetries) throws IOException {
237    Text dtService = SecurityUtil.buildTokenService(nnAddr);
238  
239    T proxy;
240    if (xface == ClientProtocol.class) {
241      proxy = (T) createNNProxyWithClientProtocol(nnAddr, conf, ugi,
242          withRetries);
243    } else if (xface == JournalProtocol.class) {
244      proxy = (T) createNNProxyWithJournalProtocol(nnAddr, conf, ugi);
245    } else if (xface == NamenodeProtocol.class) {
246      proxy = (T) createNNProxyWithNamenodeProtocol(nnAddr, conf, ugi,
247          withRetries);
248    } else if (xface == GetUserMappingsProtocol.class) {
249      proxy = (T) createNNProxyWithGetUserMappingsProtocol(nnAddr, conf, ugi);
250    } else if (xface == RefreshUserMappingsProtocol.class) {
251      proxy = (T) createNNProxyWithRefreshUserMappingsProtocol(nnAddr, conf, ugi);
252    } else if (xface == RefreshAuthorizationPolicyProtocol.class) {
253      proxy = (T) createNNProxyWithRefreshAuthorizationPolicyProtocol(nnAddr,
254          conf, ugi);
255    } else {
256      String message = "Upsupported protocol found when creating the proxy " +
257          "connection to NameNode: " +
258          ((xface != null) ? xface.getClass().getName() : "null");
259      LOG.error(message);
260      throw new IllegalStateException(message);
261    }
262    return new ProxyAndInfo<T>(proxy, dtService);
263  }
264  
265  private static JournalProtocol createNNProxyWithJournalProtocol(
266      InetSocketAddress address, Configuration conf, UserGroupInformation ugi)
267      throws IOException {
268    JournalProtocolPB proxy = (JournalProtocolPB) createNameNodeProxy(address,
269        conf, ugi, JournalProtocolPB.class);
270    return new JournalProtocolTranslatorPB(proxy);
271  }
272
273  private static RefreshAuthorizationPolicyProtocol
274      createNNProxyWithRefreshAuthorizationPolicyProtocol(InetSocketAddress address,
275          Configuration conf, UserGroupInformation ugi) throws IOException {
276    RefreshAuthorizationPolicyProtocolPB proxy = (RefreshAuthorizationPolicyProtocolPB)
277        createNameNodeProxy(address, conf, ugi, RefreshAuthorizationPolicyProtocolPB.class);
278    return new RefreshAuthorizationPolicyProtocolClientSideTranslatorPB(proxy);
279  }
280  
281  private static RefreshUserMappingsProtocol
282      createNNProxyWithRefreshUserMappingsProtocol(InetSocketAddress address,
283          Configuration conf, UserGroupInformation ugi) throws IOException {
284    RefreshUserMappingsProtocolPB proxy = (RefreshUserMappingsProtocolPB)
285        createNameNodeProxy(address, conf, ugi, RefreshUserMappingsProtocolPB.class);
286    return new RefreshUserMappingsProtocolClientSideTranslatorPB(proxy);
287  }
288
289  private static GetUserMappingsProtocol createNNProxyWithGetUserMappingsProtocol(
290      InetSocketAddress address, Configuration conf, UserGroupInformation ugi)
291      throws IOException {
292    GetUserMappingsProtocolPB proxy = (GetUserMappingsProtocolPB)
293        createNameNodeProxy(address, conf, ugi, GetUserMappingsProtocolPB.class);
294    return new GetUserMappingsProtocolClientSideTranslatorPB(proxy);
295  }
296  
297  private static NamenodeProtocol createNNProxyWithNamenodeProtocol(
298      InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
299      boolean withRetries) throws IOException {
300    NamenodeProtocolPB proxy = (NamenodeProtocolPB) createNameNodeProxy(
301        address, conf, ugi, NamenodeProtocolPB.class);
302    if (withRetries) { // create the proxy with retries
303      RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(5, 200,
304          TimeUnit.MILLISECONDS);
305      Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap 
306                     = new HashMap<Class<? extends Exception>, RetryPolicy>();
307      RetryPolicy methodPolicy = RetryPolicies.retryByException(timeoutPolicy,
308          exceptionToPolicyMap);
309      Map<String, RetryPolicy> methodNameToPolicyMap 
310                     = new HashMap<String, RetryPolicy>();
311      methodNameToPolicyMap.put("getBlocks", methodPolicy);
312      methodNameToPolicyMap.put("getAccessKeys", methodPolicy);
313      proxy = (NamenodeProtocolPB) RetryProxy.create(NamenodeProtocolPB.class,
314          proxy, methodNameToPolicyMap);
315    }
316    return new NamenodeProtocolTranslatorPB(proxy);
317  }
318  
319  private static ClientProtocol createNNProxyWithClientProtocol(
320      InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
321      boolean withRetries) throws IOException {
322    RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class);
323
324    final RetryPolicy defaultPolicy = 
325        RetryUtils.getDefaultRetryPolicy(
326            conf, 
327            DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, 
328            DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT, 
329            DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY,
330            DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
331            SafeModeException.class);
332    
333    final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class);
334    ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy(
335        ClientNamenodeProtocolPB.class, version, address, ugi, conf,
336        NetUtils.getDefaultSocketFactory(conf),
337        org.apache.hadoop.ipc.Client.getTimeout(conf), defaultPolicy)
338            .getProxy();
339
340    if (withRetries) { // create the proxy with retries
341
342      RetryPolicy createPolicy = RetryPolicies
343          .retryUpToMaximumCountWithFixedSleep(5,
344              HdfsConstants.LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
345    
346      Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap 
347                 = new HashMap<Class<? extends Exception>, RetryPolicy>();
348      remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class,
349          createPolicy);
350    
351      Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap
352                 = new HashMap<Class<? extends Exception>, RetryPolicy>();
353      exceptionToPolicyMap.put(RemoteException.class, RetryPolicies
354          .retryByRemoteException(defaultPolicy,
355              remoteExceptionToPolicyMap));
356      RetryPolicy methodPolicy = RetryPolicies.retryByException(
357          defaultPolicy, exceptionToPolicyMap);
358      Map<String, RetryPolicy> methodNameToPolicyMap 
359                 = new HashMap<String, RetryPolicy>();
360    
361      methodNameToPolicyMap.put("create", methodPolicy);
362    
363      proxy = (ClientNamenodeProtocolPB) RetryProxy.create(
364          ClientNamenodeProtocolPB.class,
365          new DefaultFailoverProxyProvider<ClientNamenodeProtocolPB>(
366              ClientNamenodeProtocolPB.class, proxy),
367          methodNameToPolicyMap,
368          defaultPolicy);
369    }
370    return new ClientNamenodeProtocolTranslatorPB(proxy);
371  }
372  
373  private static Object createNameNodeProxy(InetSocketAddress address,
374      Configuration conf, UserGroupInformation ugi, Class<?> xface)
375      throws IOException {
376    RPC.setProtocolEngine(conf, xface, ProtobufRpcEngine.class);
377    Object proxy = RPC.getProxy(xface, RPC.getProtocolVersion(xface), address,
378        ugi, conf, NetUtils.getDefaultSocketFactory(conf));
379    return proxy;
380  }
381
382  /** Gets the configured Failover proxy provider's class */
383  @VisibleForTesting
384  public static <T> Class<FailoverProxyProvider<T>> getFailoverProxyProviderClass(
385      Configuration conf, URI nameNodeUri, Class<T> xface) throws IOException {
386    if (nameNodeUri == null) {
387      return null;
388    }
389    String host = nameNodeUri.getHost();
390  
391    String configKey = DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX + "."
392        + host;
393    try {
394      @SuppressWarnings("unchecked")
395      Class<FailoverProxyProvider<T>> ret = (Class<FailoverProxyProvider<T>>) conf
396          .getClass(configKey, null, FailoverProxyProvider.class);
397      if (ret != null) {
398        // If we found a proxy provider, then this URI should be a logical NN.
399        // Given that, it shouldn't have a non-default port number.
400        int port = nameNodeUri.getPort();
401        if (port > 0 && port != NameNode.DEFAULT_PORT) {
402          throw new IOException("Port " + port + " specified in URI "
403              + nameNodeUri + " but host '" + host
404              + "' is a logical (HA) namenode"
405              + " and does not use port information.");
406        }
407      }
408      return ret;
409    } catch (RuntimeException e) {
410      if (e.getCause() instanceof ClassNotFoundException) {
411        throw new IOException("Could not load failover proxy provider class "
412            + conf.get(configKey) + " which is configured for authority "
413            + nameNodeUri, e);
414      } else {
415        throw e;
416      }
417    }
418  }
419
420  /** Creates the Failover proxy provider instance*/
421  @VisibleForTesting
422  public static <T> FailoverProxyProvider<T> createFailoverProxyProvider(
423      Configuration conf, Class<FailoverProxyProvider<T>> failoverProxyProviderClass,
424      Class<T> xface, URI nameNodeUri) throws IOException {
425    Preconditions.checkArgument(
426        xface.isAssignableFrom(NamenodeProtocols.class),
427        "Interface %s is not a NameNode protocol", xface);
428    try {
429      Constructor<FailoverProxyProvider<T>> ctor = failoverProxyProviderClass
430          .getConstructor(Configuration.class, URI.class, Class.class);
431      FailoverProxyProvider<T> provider = ctor.newInstance(conf, nameNodeUri,
432          xface);
433      return provider;
434    } catch (Exception e) {
435      String message = "Couldn't create proxy provider " + failoverProxyProviderClass;
436      if (LOG.isDebugEnabled()) {
437        LOG.debug(message, e);
438      }
439      if (e.getCause() instanceof IOException) {
440        throw (IOException) e.getCause();
441      } else {
442        throw new IOException(message, e);
443      }
444    }
445  }
446
447}