001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.hdfs.security.token.block;
020
021import java.io.ByteArrayInputStream;
022import java.io.DataInputStream;
023import java.io.IOException;
024import java.security.SecureRandom;
025import java.util.Arrays;
026import java.util.EnumSet;
027import java.util.HashMap;
028import java.util.Iterator;
029import java.util.Map;
030
031import org.apache.commons.logging.Log;
032import org.apache.commons.logging.LogFactory;
033import org.apache.hadoop.classification.InterfaceAudience;
034import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
035import org.apache.hadoop.hdfs.protocol.datatransfer.InvalidEncryptionKeyException;
036import org.apache.hadoop.io.WritableUtils;
037import org.apache.hadoop.security.UserGroupInformation;
038import org.apache.hadoop.security.token.SecretManager;
039import org.apache.hadoop.security.token.Token;
040import org.apache.hadoop.util.Time;
041
042import com.google.common.annotations.VisibleForTesting;
043import com.google.common.base.Preconditions;
044
045/**
046 * BlockTokenSecretManager can be instantiated in 2 modes, master mode and slave
047 * mode. Master can generate new block keys and export block keys to slaves,
048 * while slaves can only import and use block keys received from master. Both
049 * master and slave can generate and verify block tokens. Typically, master mode
050 * is used by NN and slave mode is used by DN.
051 */
052@InterfaceAudience.Private
053public class BlockTokenSecretManager extends
054    SecretManager<BlockTokenIdentifier> {
055  public static final Log LOG = LogFactory
056      .getLog(BlockTokenSecretManager.class);
057  
058  // We use these in an HA setup to ensure that the pair of NNs produce block
059  // token serial numbers that are in different ranges.
060  private static final int LOW_MASK  = ~(1 << 31);
061  
062  public static final Token<BlockTokenIdentifier> DUMMY_TOKEN = new Token<BlockTokenIdentifier>();
063
064  private final boolean isMaster;
065  private int nnIndex;
066  
067  /**
068   * keyUpdateInterval is the interval that NN updates its block keys. It should
069   * be set long enough so that all live DN's and Balancer should have sync'ed
070   * their block keys with NN at least once during each interval.
071   */
072  private long keyUpdateInterval;
073  private volatile long tokenLifetime;
074  private int serialNo;
075  private BlockKey currentKey;
076  private BlockKey nextKey;
077  private Map<Integer, BlockKey> allKeys;
078  private String blockPoolId;
079  private String encryptionAlgorithm;
080  
081  private SecureRandom nonceGenerator = new SecureRandom();
082
083  public static enum AccessMode {
084    READ, WRITE, COPY, REPLACE
085  };
086  
087  /**
088   * Constructor for slaves.
089   * 
090   * @param keyUpdateInterval how often a new key will be generated
091   * @param tokenLifetime how long an individual token is valid
092   */
093  public BlockTokenSecretManager(long keyUpdateInterval,
094      long tokenLifetime, String blockPoolId, String encryptionAlgorithm) {
095    this(false, keyUpdateInterval, tokenLifetime, blockPoolId,
096        encryptionAlgorithm);
097  }
098  
099  /**
100   * Constructor for masters.
101   * 
102   * @param keyUpdateInterval how often a new key will be generated
103   * @param tokenLifetime how long an individual token is valid
104   * @param isHaEnabled whether or not HA is enabled
105   * @param thisNnId the NN ID of this NN in an HA setup
106   * @param otherNnId the NN ID of the other NN in an HA setup
107   */
108  public BlockTokenSecretManager(long keyUpdateInterval,
109      long tokenLifetime, int nnIndex, String blockPoolId,
110      String encryptionAlgorithm) {
111    this(true, keyUpdateInterval, tokenLifetime, blockPoolId,
112        encryptionAlgorithm);
113    Preconditions.checkArgument(nnIndex == 0 || nnIndex == 1);
114    this.nnIndex = nnIndex;
115    setSerialNo(new SecureRandom().nextInt());
116    generateKeys();
117  }
118  
119  private BlockTokenSecretManager(boolean isMaster, long keyUpdateInterval,
120      long tokenLifetime, String blockPoolId, String encryptionAlgorithm) {
121    this.isMaster = isMaster;
122    this.keyUpdateInterval = keyUpdateInterval;
123    this.tokenLifetime = tokenLifetime;
124    this.allKeys = new HashMap<Integer, BlockKey>();
125    this.blockPoolId = blockPoolId;
126    this.encryptionAlgorithm = encryptionAlgorithm;
127    generateKeys();
128  }
129  
130  @VisibleForTesting
131  public synchronized void setSerialNo(int serialNo) {
132    this.serialNo = (serialNo & LOW_MASK) | (nnIndex << 31);
133  }
134  
135  public void setBlockPoolId(String blockPoolId) {
136    this.blockPoolId = blockPoolId;
137  }
138
139  /** Initialize block keys */
140  private synchronized void generateKeys() {
141    if (!isMaster)
142      return;
143    /*
144     * Need to set estimated expiry dates for currentKey and nextKey so that if
145     * NN crashes, DN can still expire those keys. NN will stop using the newly
146     * generated currentKey after the first keyUpdateInterval, however it may
147     * still be used by DN and Balancer to generate new tokens before they get a
148     * chance to sync their keys with NN. Since we require keyUpdInterval to be
149     * long enough so that all live DN's and Balancer will sync their keys with
150     * NN at least once during the period, the estimated expiry date for
151     * currentKey is set to now() + 2 * keyUpdateInterval + tokenLifetime.
152     * Similarly, the estimated expiry date for nextKey is one keyUpdateInterval
153     * more.
154     */
155    setSerialNo(serialNo + 1);
156    currentKey = new BlockKey(serialNo, Time.now() + 2
157        * keyUpdateInterval + tokenLifetime, generateSecret());
158    setSerialNo(serialNo + 1);
159    nextKey = new BlockKey(serialNo, Time.now() + 3
160        * keyUpdateInterval + tokenLifetime, generateSecret());
161    allKeys.put(currentKey.getKeyId(), currentKey);
162    allKeys.put(nextKey.getKeyId(), nextKey);
163  }
164
165  /** Export block keys, only to be used in master mode */
166  public synchronized ExportedBlockKeys exportKeys() {
167    if (!isMaster)
168      return null;
169    if (LOG.isDebugEnabled())
170      LOG.debug("Exporting access keys");
171    return new ExportedBlockKeys(true, keyUpdateInterval, tokenLifetime,
172        currentKey, allKeys.values().toArray(new BlockKey[0]));
173  }
174
175  private synchronized void removeExpiredKeys() {
176    long now = Time.now();
177    for (Iterator<Map.Entry<Integer, BlockKey>> it = allKeys.entrySet()
178        .iterator(); it.hasNext();) {
179      Map.Entry<Integer, BlockKey> e = it.next();
180      if (e.getValue().getExpiryDate() < now) {
181        it.remove();
182      }
183    }
184  }
185
186  /**
187   * Set block keys, only to be used in slave mode
188   */
189  public synchronized void addKeys(ExportedBlockKeys exportedKeys)
190      throws IOException {
191    if (isMaster || exportedKeys == null)
192      return;
193    LOG.info("Setting block keys");
194    removeExpiredKeys();
195    this.currentKey = exportedKeys.getCurrentKey();
196    BlockKey[] receivedKeys = exportedKeys.getAllKeys();
197    for (int i = 0; i < receivedKeys.length; i++) {
198      if (receivedKeys[i] == null)
199        continue;
200      this.allKeys.put(receivedKeys[i].getKeyId(), receivedKeys[i]);
201    }
202  }
203
204  /**
205   * Update block keys if update time > update interval.
206   * @return true if the keys are updated.
207   */
208  public synchronized boolean updateKeys(final long updateTime) throws IOException {
209    if (updateTime > keyUpdateInterval) {
210      return updateKeys();
211    }
212    return false;
213  }
214
215  /**
216   * Update block keys, only to be used in master mode
217   */
218  synchronized boolean updateKeys() throws IOException {
219    if (!isMaster)
220      return false;
221
222    LOG.info("Updating block keys");
223    removeExpiredKeys();
224    // set final expiry date of retiring currentKey
225    allKeys.put(currentKey.getKeyId(), new BlockKey(currentKey.getKeyId(),
226        Time.now() + keyUpdateInterval + tokenLifetime,
227        currentKey.getKey()));
228    // update the estimated expiry date of new currentKey
229    currentKey = new BlockKey(nextKey.getKeyId(), Time.now()
230        + 2 * keyUpdateInterval + tokenLifetime, nextKey.getKey());
231    allKeys.put(currentKey.getKeyId(), currentKey);
232    // generate a new nextKey
233    setSerialNo(serialNo + 1);
234    nextKey = new BlockKey(serialNo, Time.now() + 3
235        * keyUpdateInterval + tokenLifetime, generateSecret());
236    allKeys.put(nextKey.getKeyId(), nextKey);
237    return true;
238  }
239
240  /** Generate an block token for current user */
241  public Token<BlockTokenIdentifier> generateToken(ExtendedBlock block,
242      EnumSet<AccessMode> modes) throws IOException {
243    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
244    String userID = (ugi == null ? null : ugi.getShortUserName());
245    return generateToken(userID, block, modes);
246  }
247
248  /** Generate a block token for a specified user */
249  public Token<BlockTokenIdentifier> generateToken(String userId,
250      ExtendedBlock block, EnumSet<AccessMode> modes) throws IOException {
251    BlockTokenIdentifier id = new BlockTokenIdentifier(userId, block
252        .getBlockPoolId(), block.getBlockId(), modes);
253    return new Token<BlockTokenIdentifier>(id, this);
254  }
255
256  /**
257   * Check if access should be allowed. userID is not checked if null. This
258   * method doesn't check if token password is correct. It should be used only
259   * when token password has already been verified (e.g., in the RPC layer).
260   */
261  public void checkAccess(BlockTokenIdentifier id, String userId,
262      ExtendedBlock block, AccessMode mode) throws InvalidToken {
263    if (LOG.isDebugEnabled()) {
264      LOG.debug("Checking access for user=" + userId + ", block=" + block
265          + ", access mode=" + mode + " using " + id.toString());
266    }
267    if (userId != null && !userId.equals(id.getUserId())) {
268      throw new InvalidToken("Block token with " + id.toString()
269          + " doesn't belong to user " + userId);
270    }
271    if (!id.getBlockPoolId().equals(block.getBlockPoolId())) {
272      throw new InvalidToken("Block token with " + id.toString()
273          + " doesn't apply to block " + block);
274    }
275    if (id.getBlockId() != block.getBlockId()) {
276      throw new InvalidToken("Block token with " + id.toString()
277          + " doesn't apply to block " + block);
278    }
279    if (isExpired(id.getExpiryDate())) {
280      throw new InvalidToken("Block token with " + id.toString()
281          + " is expired.");
282    }
283    if (!id.getAccessModes().contains(mode)) {
284      throw new InvalidToken("Block token with " + id.toString()
285          + " doesn't have " + mode + " permission");
286    }
287  }
288
289  /** Check if access should be allowed. userID is not checked if null */
290  public void checkAccess(Token<BlockTokenIdentifier> token, String userId,
291      ExtendedBlock block, AccessMode mode) throws InvalidToken {
292    BlockTokenIdentifier id = new BlockTokenIdentifier();
293    try {
294      id.readFields(new DataInputStream(new ByteArrayInputStream(token
295          .getIdentifier())));
296    } catch (IOException e) {
297      throw new InvalidToken(
298          "Unable to de-serialize block token identifier for user=" + userId
299              + ", block=" + block + ", access mode=" + mode);
300    }
301    checkAccess(id, userId, block, mode);
302    if (!Arrays.equals(retrievePassword(id), token.getPassword())) {
303      throw new InvalidToken("Block token with " + id.toString()
304          + " doesn't have the correct token password");
305    }
306  }
307
308  private static boolean isExpired(long expiryDate) {
309    return Time.now() > expiryDate;
310  }
311
312  /**
313   * check if a token is expired. for unit test only. return true when token is
314   * expired, false otherwise
315   */
316  static boolean isTokenExpired(Token<BlockTokenIdentifier> token)
317      throws IOException {
318    ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
319    DataInputStream in = new DataInputStream(buf);
320    long expiryDate = WritableUtils.readVLong(in);
321    return isExpired(expiryDate);
322  }
323
324  /** set token lifetime. */
325  public void setTokenLifetime(long tokenLifetime) {
326    this.tokenLifetime = tokenLifetime;
327  }
328
329  /**
330   * Create an empty block token identifier
331   * 
332   * @return a newly created empty block token identifier
333   */
334  @Override
335  public BlockTokenIdentifier createIdentifier() {
336    return new BlockTokenIdentifier();
337  }
338
339  /**
340   * Create a new password/secret for the given block token identifier.
341   * 
342   * @param identifier
343   *          the block token identifier
344   * @return token password/secret
345   */
346  @Override
347  protected byte[] createPassword(BlockTokenIdentifier identifier) {
348    BlockKey key = null;
349    synchronized (this) {
350      key = currentKey;
351    }
352    if (key == null)
353      throw new IllegalStateException("currentKey hasn't been initialized.");
354    identifier.setExpiryDate(Time.now() + tokenLifetime);
355    identifier.setKeyId(key.getKeyId());
356    if (LOG.isDebugEnabled()) {
357      LOG.debug("Generating block token for " + identifier.toString());
358    }
359    return createPassword(identifier.getBytes(), key.getKey());
360  }
361
362  /**
363   * Look up the token password/secret for the given block token identifier.
364   * 
365   * @param identifier
366   *          the block token identifier to look up
367   * @return token password/secret as byte[]
368   * @throws InvalidToken
369   */
370  @Override
371  public byte[] retrievePassword(BlockTokenIdentifier identifier)
372      throws InvalidToken {
373    if (isExpired(identifier.getExpiryDate())) {
374      throw new InvalidToken("Block token with " + identifier.toString()
375          + " is expired.");
376    }
377    BlockKey key = null;
378    synchronized (this) {
379      key = allKeys.get(identifier.getKeyId());
380    }
381    if (key == null) {
382      throw new InvalidToken("Can't re-compute password for "
383          + identifier.toString() + ", since the required block key (keyID="
384          + identifier.getKeyId() + ") doesn't exist.");
385    }
386    return createPassword(identifier.getBytes(), key.getKey());
387  }
388  
389  /**
390   * Generate a data encryption key for this block pool, using the current
391   * BlockKey.
392   * 
393   * @return a data encryption key which may be used to encrypt traffic
394   *         over the DataTransferProtocol
395   */
396  public DataEncryptionKey generateDataEncryptionKey() {
397    byte[] nonce = new byte[8];
398    nonceGenerator.nextBytes(nonce);
399    BlockKey key = null;
400    synchronized (this) {
401      key = currentKey;
402    }
403    byte[] encryptionKey = createPassword(nonce, key.getKey());
404    return new DataEncryptionKey(key.getKeyId(), blockPoolId, nonce,
405        encryptionKey, Time.now() + tokenLifetime,
406        encryptionAlgorithm);
407  }
408  
409  /**
410   * Recreate an encryption key based on the given key id and nonce.
411   * 
412   * @param keyId identifier of the secret key used to generate the encryption key.
413   * @param nonce random value used to create the encryption key
414   * @return the encryption key which corresponds to this (keyId, blockPoolId, nonce)
415   * @throws InvalidToken
416   * @throws InvalidEncryptionKeyException 
417   */
418  public byte[] retrieveDataEncryptionKey(int keyId, byte[] nonce)
419      throws InvalidEncryptionKeyException {
420    BlockKey key = null;
421    synchronized (this) {
422      key = allKeys.get(keyId);
423      if (key == null) {
424        throw new InvalidEncryptionKeyException("Can't re-compute encryption key"
425            + " for nonce, since the required block key (keyID=" + keyId
426            + ") doesn't exist. Current key: " + currentKey.getKeyId());
427      }
428    }
429    return createPassword(nonce, key.getKey());
430  }
431  
432  @VisibleForTesting
433  public synchronized void setKeyUpdateIntervalForTesting(long millis) {
434    this.keyUpdateInterval = millis;
435  }
436
437  @VisibleForTesting
438  public void clearAllKeysForTesting() {
439    allKeys.clear();
440  }
441  
442  @VisibleForTesting
443  public synchronized int getSerialNoForTesting() {
444    return serialNo;
445  }
446  
447}