001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.hdfs.security.token.block; 020 021import java.io.ByteArrayInputStream; 022import java.io.DataInputStream; 023import java.io.IOException; 024import java.security.SecureRandom; 025import java.util.Arrays; 026import java.util.EnumSet; 027import java.util.HashMap; 028import java.util.Iterator; 029import java.util.Map; 030 031import org.apache.commons.logging.Log; 032import org.apache.commons.logging.LogFactory; 033import org.apache.hadoop.classification.InterfaceAudience; 034import org.apache.hadoop.hdfs.protocol.ExtendedBlock; 035import org.apache.hadoop.hdfs.protocol.datatransfer.InvalidEncryptionKeyException; 036import org.apache.hadoop.io.WritableUtils; 037import org.apache.hadoop.security.UserGroupInformation; 038import org.apache.hadoop.security.token.SecretManager; 039import org.apache.hadoop.security.token.Token; 040import org.apache.hadoop.util.Time; 041 042import com.google.common.annotations.VisibleForTesting; 043import com.google.common.base.Preconditions; 044 045/** 046 * BlockTokenSecretManager can be instantiated in 2 modes, master mode and slave 047 * mode. Master can generate new block keys and export block keys to slaves, 048 * while slaves can only import and use block keys received from master. Both 049 * master and slave can generate and verify block tokens. Typically, master mode 050 * is used by NN and slave mode is used by DN. 051 */ 052@InterfaceAudience.Private 053public class BlockTokenSecretManager extends 054 SecretManager<BlockTokenIdentifier> { 055 public static final Log LOG = LogFactory 056 .getLog(BlockTokenSecretManager.class); 057 058 // We use these in an HA setup to ensure that the pair of NNs produce block 059 // token serial numbers that are in different ranges. 060 private static final int LOW_MASK = ~(1 << 31); 061 062 public static final Token<BlockTokenIdentifier> DUMMY_TOKEN = new Token<BlockTokenIdentifier>(); 063 064 private final boolean isMaster; 065 private int nnIndex; 066 067 /** 068 * keyUpdateInterval is the interval that NN updates its block keys. It should 069 * be set long enough so that all live DN's and Balancer should have sync'ed 070 * their block keys with NN at least once during each interval. 071 */ 072 private long keyUpdateInterval; 073 private volatile long tokenLifetime; 074 private int serialNo; 075 private BlockKey currentKey; 076 private BlockKey nextKey; 077 private Map<Integer, BlockKey> allKeys; 078 private String blockPoolId; 079 private String encryptionAlgorithm; 080 081 private SecureRandom nonceGenerator = new SecureRandom(); 082 083 public static enum AccessMode { 084 READ, WRITE, COPY, REPLACE 085 }; 086 087 /** 088 * Constructor for slaves. 089 * 090 * @param keyUpdateInterval how often a new key will be generated 091 * @param tokenLifetime how long an individual token is valid 092 */ 093 public BlockTokenSecretManager(long keyUpdateInterval, 094 long tokenLifetime, String blockPoolId, String encryptionAlgorithm) { 095 this(false, keyUpdateInterval, tokenLifetime, blockPoolId, 096 encryptionAlgorithm); 097 } 098 099 /** 100 * Constructor for masters. 101 * 102 * @param keyUpdateInterval how often a new key will be generated 103 * @param tokenLifetime how long an individual token is valid 104 * @param isHaEnabled whether or not HA is enabled 105 * @param thisNnId the NN ID of this NN in an HA setup 106 * @param otherNnId the NN ID of the other NN in an HA setup 107 */ 108 public BlockTokenSecretManager(long keyUpdateInterval, 109 long tokenLifetime, int nnIndex, String blockPoolId, 110 String encryptionAlgorithm) { 111 this(true, keyUpdateInterval, tokenLifetime, blockPoolId, 112 encryptionAlgorithm); 113 Preconditions.checkArgument(nnIndex == 0 || nnIndex == 1); 114 this.nnIndex = nnIndex; 115 setSerialNo(new SecureRandom().nextInt()); 116 generateKeys(); 117 } 118 119 private BlockTokenSecretManager(boolean isMaster, long keyUpdateInterval, 120 long tokenLifetime, String blockPoolId, String encryptionAlgorithm) { 121 this.isMaster = isMaster; 122 this.keyUpdateInterval = keyUpdateInterval; 123 this.tokenLifetime = tokenLifetime; 124 this.allKeys = new HashMap<Integer, BlockKey>(); 125 this.blockPoolId = blockPoolId; 126 this.encryptionAlgorithm = encryptionAlgorithm; 127 generateKeys(); 128 } 129 130 @VisibleForTesting 131 public synchronized void setSerialNo(int serialNo) { 132 this.serialNo = (serialNo & LOW_MASK) | (nnIndex << 31); 133 } 134 135 public void setBlockPoolId(String blockPoolId) { 136 this.blockPoolId = blockPoolId; 137 } 138 139 /** Initialize block keys */ 140 private synchronized void generateKeys() { 141 if (!isMaster) 142 return; 143 /* 144 * Need to set estimated expiry dates for currentKey and nextKey so that if 145 * NN crashes, DN can still expire those keys. NN will stop using the newly 146 * generated currentKey after the first keyUpdateInterval, however it may 147 * still be used by DN and Balancer to generate new tokens before they get a 148 * chance to sync their keys with NN. Since we require keyUpdInterval to be 149 * long enough so that all live DN's and Balancer will sync their keys with 150 * NN at least once during the period, the estimated expiry date for 151 * currentKey is set to now() + 2 * keyUpdateInterval + tokenLifetime. 152 * Similarly, the estimated expiry date for nextKey is one keyUpdateInterval 153 * more. 154 */ 155 setSerialNo(serialNo + 1); 156 currentKey = new BlockKey(serialNo, Time.now() + 2 157 * keyUpdateInterval + tokenLifetime, generateSecret()); 158 setSerialNo(serialNo + 1); 159 nextKey = new BlockKey(serialNo, Time.now() + 3 160 * keyUpdateInterval + tokenLifetime, generateSecret()); 161 allKeys.put(currentKey.getKeyId(), currentKey); 162 allKeys.put(nextKey.getKeyId(), nextKey); 163 } 164 165 /** Export block keys, only to be used in master mode */ 166 public synchronized ExportedBlockKeys exportKeys() { 167 if (!isMaster) 168 return null; 169 if (LOG.isDebugEnabled()) 170 LOG.debug("Exporting access keys"); 171 return new ExportedBlockKeys(true, keyUpdateInterval, tokenLifetime, 172 currentKey, allKeys.values().toArray(new BlockKey[0])); 173 } 174 175 private synchronized void removeExpiredKeys() { 176 long now = Time.now(); 177 for (Iterator<Map.Entry<Integer, BlockKey>> it = allKeys.entrySet() 178 .iterator(); it.hasNext();) { 179 Map.Entry<Integer, BlockKey> e = it.next(); 180 if (e.getValue().getExpiryDate() < now) { 181 it.remove(); 182 } 183 } 184 } 185 186 /** 187 * Set block keys, only to be used in slave mode 188 */ 189 public synchronized void addKeys(ExportedBlockKeys exportedKeys) 190 throws IOException { 191 if (isMaster || exportedKeys == null) 192 return; 193 LOG.info("Setting block keys"); 194 removeExpiredKeys(); 195 this.currentKey = exportedKeys.getCurrentKey(); 196 BlockKey[] receivedKeys = exportedKeys.getAllKeys(); 197 for (int i = 0; i < receivedKeys.length; i++) { 198 if (receivedKeys[i] == null) 199 continue; 200 this.allKeys.put(receivedKeys[i].getKeyId(), receivedKeys[i]); 201 } 202 } 203 204 /** 205 * Update block keys if update time > update interval. 206 * @return true if the keys are updated. 207 */ 208 public synchronized boolean updateKeys(final long updateTime) throws IOException { 209 if (updateTime > keyUpdateInterval) { 210 return updateKeys(); 211 } 212 return false; 213 } 214 215 /** 216 * Update block keys, only to be used in master mode 217 */ 218 synchronized boolean updateKeys() throws IOException { 219 if (!isMaster) 220 return false; 221 222 LOG.info("Updating block keys"); 223 removeExpiredKeys(); 224 // set final expiry date of retiring currentKey 225 allKeys.put(currentKey.getKeyId(), new BlockKey(currentKey.getKeyId(), 226 Time.now() + keyUpdateInterval + tokenLifetime, 227 currentKey.getKey())); 228 // update the estimated expiry date of new currentKey 229 currentKey = new BlockKey(nextKey.getKeyId(), Time.now() 230 + 2 * keyUpdateInterval + tokenLifetime, nextKey.getKey()); 231 allKeys.put(currentKey.getKeyId(), currentKey); 232 // generate a new nextKey 233 setSerialNo(serialNo + 1); 234 nextKey = new BlockKey(serialNo, Time.now() + 3 235 * keyUpdateInterval + tokenLifetime, generateSecret()); 236 allKeys.put(nextKey.getKeyId(), nextKey); 237 return true; 238 } 239 240 /** Generate an block token for current user */ 241 public Token<BlockTokenIdentifier> generateToken(ExtendedBlock block, 242 EnumSet<AccessMode> modes) throws IOException { 243 UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); 244 String userID = (ugi == null ? null : ugi.getShortUserName()); 245 return generateToken(userID, block, modes); 246 } 247 248 /** Generate a block token for a specified user */ 249 public Token<BlockTokenIdentifier> generateToken(String userId, 250 ExtendedBlock block, EnumSet<AccessMode> modes) throws IOException { 251 BlockTokenIdentifier id = new BlockTokenIdentifier(userId, block 252 .getBlockPoolId(), block.getBlockId(), modes); 253 return new Token<BlockTokenIdentifier>(id, this); 254 } 255 256 /** 257 * Check if access should be allowed. userID is not checked if null. This 258 * method doesn't check if token password is correct. It should be used only 259 * when token password has already been verified (e.g., in the RPC layer). 260 */ 261 public void checkAccess(BlockTokenIdentifier id, String userId, 262 ExtendedBlock block, AccessMode mode) throws InvalidToken { 263 if (LOG.isDebugEnabled()) { 264 LOG.debug("Checking access for user=" + userId + ", block=" + block 265 + ", access mode=" + mode + " using " + id.toString()); 266 } 267 if (userId != null && !userId.equals(id.getUserId())) { 268 throw new InvalidToken("Block token with " + id.toString() 269 + " doesn't belong to user " + userId); 270 } 271 if (!id.getBlockPoolId().equals(block.getBlockPoolId())) { 272 throw new InvalidToken("Block token with " + id.toString() 273 + " doesn't apply to block " + block); 274 } 275 if (id.getBlockId() != block.getBlockId()) { 276 throw new InvalidToken("Block token with " + id.toString() 277 + " doesn't apply to block " + block); 278 } 279 if (isExpired(id.getExpiryDate())) { 280 throw new InvalidToken("Block token with " + id.toString() 281 + " is expired."); 282 } 283 if (!id.getAccessModes().contains(mode)) { 284 throw new InvalidToken("Block token with " + id.toString() 285 + " doesn't have " + mode + " permission"); 286 } 287 } 288 289 /** Check if access should be allowed. userID is not checked if null */ 290 public void checkAccess(Token<BlockTokenIdentifier> token, String userId, 291 ExtendedBlock block, AccessMode mode) throws InvalidToken { 292 BlockTokenIdentifier id = new BlockTokenIdentifier(); 293 try { 294 id.readFields(new DataInputStream(new ByteArrayInputStream(token 295 .getIdentifier()))); 296 } catch (IOException e) { 297 throw new InvalidToken( 298 "Unable to de-serialize block token identifier for user=" + userId 299 + ", block=" + block + ", access mode=" + mode); 300 } 301 checkAccess(id, userId, block, mode); 302 if (!Arrays.equals(retrievePassword(id), token.getPassword())) { 303 throw new InvalidToken("Block token with " + id.toString() 304 + " doesn't have the correct token password"); 305 } 306 } 307 308 private static boolean isExpired(long expiryDate) { 309 return Time.now() > expiryDate; 310 } 311 312 /** 313 * check if a token is expired. for unit test only. return true when token is 314 * expired, false otherwise 315 */ 316 static boolean isTokenExpired(Token<BlockTokenIdentifier> token) 317 throws IOException { 318 ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); 319 DataInputStream in = new DataInputStream(buf); 320 long expiryDate = WritableUtils.readVLong(in); 321 return isExpired(expiryDate); 322 } 323 324 /** set token lifetime. */ 325 public void setTokenLifetime(long tokenLifetime) { 326 this.tokenLifetime = tokenLifetime; 327 } 328 329 /** 330 * Create an empty block token identifier 331 * 332 * @return a newly created empty block token identifier 333 */ 334 @Override 335 public BlockTokenIdentifier createIdentifier() { 336 return new BlockTokenIdentifier(); 337 } 338 339 /** 340 * Create a new password/secret for the given block token identifier. 341 * 342 * @param identifier 343 * the block token identifier 344 * @return token password/secret 345 */ 346 @Override 347 protected byte[] createPassword(BlockTokenIdentifier identifier) { 348 BlockKey key = null; 349 synchronized (this) { 350 key = currentKey; 351 } 352 if (key == null) 353 throw new IllegalStateException("currentKey hasn't been initialized."); 354 identifier.setExpiryDate(Time.now() + tokenLifetime); 355 identifier.setKeyId(key.getKeyId()); 356 if (LOG.isDebugEnabled()) { 357 LOG.debug("Generating block token for " + identifier.toString()); 358 } 359 return createPassword(identifier.getBytes(), key.getKey()); 360 } 361 362 /** 363 * Look up the token password/secret for the given block token identifier. 364 * 365 * @param identifier 366 * the block token identifier to look up 367 * @return token password/secret as byte[] 368 * @throws InvalidToken 369 */ 370 @Override 371 public byte[] retrievePassword(BlockTokenIdentifier identifier) 372 throws InvalidToken { 373 if (isExpired(identifier.getExpiryDate())) { 374 throw new InvalidToken("Block token with " + identifier.toString() 375 + " is expired."); 376 } 377 BlockKey key = null; 378 synchronized (this) { 379 key = allKeys.get(identifier.getKeyId()); 380 } 381 if (key == null) { 382 throw new InvalidToken("Can't re-compute password for " 383 + identifier.toString() + ", since the required block key (keyID=" 384 + identifier.getKeyId() + ") doesn't exist."); 385 } 386 return createPassword(identifier.getBytes(), key.getKey()); 387 } 388 389 /** 390 * Generate a data encryption key for this block pool, using the current 391 * BlockKey. 392 * 393 * @return a data encryption key which may be used to encrypt traffic 394 * over the DataTransferProtocol 395 */ 396 public DataEncryptionKey generateDataEncryptionKey() { 397 byte[] nonce = new byte[8]; 398 nonceGenerator.nextBytes(nonce); 399 BlockKey key = null; 400 synchronized (this) { 401 key = currentKey; 402 } 403 byte[] encryptionKey = createPassword(nonce, key.getKey()); 404 return new DataEncryptionKey(key.getKeyId(), blockPoolId, nonce, 405 encryptionKey, Time.now() + tokenLifetime, 406 encryptionAlgorithm); 407 } 408 409 /** 410 * Recreate an encryption key based on the given key id and nonce. 411 * 412 * @param keyId identifier of the secret key used to generate the encryption key. 413 * @param nonce random value used to create the encryption key 414 * @return the encryption key which corresponds to this (keyId, blockPoolId, nonce) 415 * @throws InvalidToken 416 * @throws InvalidEncryptionKeyException 417 */ 418 public byte[] retrieveDataEncryptionKey(int keyId, byte[] nonce) 419 throws InvalidEncryptionKeyException { 420 BlockKey key = null; 421 synchronized (this) { 422 key = allKeys.get(keyId); 423 if (key == null) { 424 throw new InvalidEncryptionKeyException("Can't re-compute encryption key" 425 + " for nonce, since the required block key (keyID=" + keyId 426 + ") doesn't exist. Current key: " + currentKey.getKeyId()); 427 } 428 } 429 return createPassword(nonce, key.getKey()); 430 } 431 432 @VisibleForTesting 433 public synchronized void setKeyUpdateIntervalForTesting(long millis) { 434 this.keyUpdateInterval = millis; 435 } 436 437 @VisibleForTesting 438 public void clearAllKeysForTesting() { 439 allKeys.clear(); 440 } 441 442 @VisibleForTesting 443 public synchronized int getSerialNoForTesting() { 444 return serialNo; 445 } 446 447}