001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hdfs.server.blockmanagement; 019 020import java.util.Arrays; 021import java.util.Iterator; 022import java.util.List; 023 024import com.google.common.annotations.VisibleForTesting; 025import org.apache.hadoop.hdfs.StorageType; 026import org.apache.hadoop.hdfs.protocol.DatanodeInfo; 027import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; 028import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State; 029import org.apache.hadoop.hdfs.server.protocol.StorageReport; 030 031/** 032 * A Datanode has one or more storages. A storage in the Datanode is represented 033 * by this class. 034 */ 035public class DatanodeStorageInfo { 036 public static final DatanodeStorageInfo[] EMPTY_ARRAY = {}; 037 038 public static DatanodeInfo[] toDatanodeInfos(DatanodeStorageInfo[] storages) { 039 return toDatanodeInfos(Arrays.asList(storages)); 040 } 041 static DatanodeInfo[] toDatanodeInfos(List<DatanodeStorageInfo> storages) { 042 final DatanodeInfo[] datanodes = new DatanodeInfo[storages.size()]; 043 for(int i = 0; i < storages.size(); i++) { 044 datanodes[i] = storages.get(i).getDatanodeDescriptor(); 045 } 046 return datanodes; 047 } 048 049 static DatanodeDescriptor[] toDatanodeDescriptors( 050 DatanodeStorageInfo[] storages) { 051 DatanodeDescriptor[] datanodes = new DatanodeDescriptor[storages.length]; 052 for (int i = 0; i < storages.length; ++i) { 053 datanodes[i] = storages[i].getDatanodeDescriptor(); 054 } 055 return datanodes; 056 } 057 058 public static String[] toStorageIDs(DatanodeStorageInfo[] storages) { 059 String[] storageIDs = new String[storages.length]; 060 for(int i = 0; i < storageIDs.length; i++) { 061 storageIDs[i] = storages[i].getStorageID(); 062 } 063 return storageIDs; 064 } 065 066 public static StorageType[] toStorageTypes(DatanodeStorageInfo[] storages) { 067 StorageType[] storageTypes = new StorageType[storages.length]; 068 for(int i = 0; i < storageTypes.length; i++) { 069 storageTypes[i] = storages[i].getStorageType(); 070 } 071 return storageTypes; 072 } 073 074 /** 075 * Iterates over the list of blocks belonging to the data-node. 076 */ 077 class BlockIterator implements Iterator<BlockInfo> { 078 private BlockInfo current; 079 080 BlockIterator(BlockInfo head) { 081 this.current = head; 082 } 083 084 public boolean hasNext() { 085 return current != null; 086 } 087 088 public BlockInfo next() { 089 BlockInfo res = current; 090 current = current.getNext(current.findStorageInfo(DatanodeStorageInfo.this)); 091 return res; 092 } 093 094 public void remove() { 095 throw new UnsupportedOperationException("Sorry. can't remove."); 096 } 097 } 098 099 private final DatanodeDescriptor dn; 100 private final String storageID; 101 private final StorageType storageType; 102 private final State state; 103 104 private long capacity; 105 private long dfsUsed; 106 private long remaining; 107 private long blockPoolUsed; 108 109 private volatile BlockInfo blockList = null; 110 private int numBlocks = 0; 111 112 /** The number of block reports received */ 113 private int blockReportCount = 0; 114 115 /** 116 * Set to false on any NN failover, and reset to true 117 * whenever a block report is received. 118 */ 119 private boolean heartbeatedSinceFailover = false; 120 121 /** 122 * At startup or at failover, the storages in the cluster may have pending 123 * block deletions from a previous incarnation of the NameNode. The block 124 * contents are considered as stale until a block report is received. When a 125 * storage is considered as stale, the replicas on it are also considered as 126 * stale. If any block has at least one stale replica, then no invalidations 127 * will be processed for this block. See HDFS-1972. 128 */ 129 private boolean blockContentsStale = true; 130 131 DatanodeStorageInfo(DatanodeDescriptor dn, DatanodeStorage s) { 132 this.dn = dn; 133 this.storageID = s.getStorageID(); 134 this.storageType = s.getStorageType(); 135 this.state = s.getState(); 136 } 137 138 int getBlockReportCount() { 139 return blockReportCount; 140 } 141 142 void setBlockReportCount(int blockReportCount) { 143 this.blockReportCount = blockReportCount; 144 } 145 146 boolean areBlockContentsStale() { 147 return blockContentsStale; 148 } 149 150 void markStaleAfterFailover() { 151 heartbeatedSinceFailover = false; 152 blockContentsStale = true; 153 } 154 155 void receivedHeartbeat(StorageReport report) { 156 updateState(report); 157 heartbeatedSinceFailover = true; 158 } 159 160 void receivedBlockReport() { 161 if (heartbeatedSinceFailover) { 162 blockContentsStale = false; 163 } 164 blockReportCount++; 165 } 166 167 @VisibleForTesting 168 public void setUtilizationForTesting(long capacity, long dfsUsed, 169 long remaining, long blockPoolUsed) { 170 this.capacity = capacity; 171 this.dfsUsed = dfsUsed; 172 this.remaining = remaining; 173 this.blockPoolUsed = blockPoolUsed; 174 } 175 176 State getState() { 177 return this.state; 178 } 179 180 String getStorageID() { 181 return storageID; 182 } 183 184 StorageType getStorageType() { 185 return storageType; 186 } 187 188 long getCapacity() { 189 return capacity; 190 } 191 192 long getDfsUsed() { 193 return dfsUsed; 194 } 195 196 long getRemaining() { 197 return remaining; 198 } 199 200 long getBlockPoolUsed() { 201 return blockPoolUsed; 202 } 203 204 boolean addBlock(BlockInfo b) { 205 if(!b.addStorage(this)) 206 return false; 207 // add to the head of the data-node list 208 blockList = b.listInsert(blockList, this); 209 numBlocks++; 210 return true; 211 } 212 213 boolean removeBlock(BlockInfo b) { 214 blockList = b.listRemove(blockList, this); 215 if (b.removeStorage(this)) { 216 numBlocks--; 217 return true; 218 } else { 219 return false; 220 } 221 } 222 223 int numBlocks() { 224 return numBlocks; 225 } 226 227 Iterator<BlockInfo> getBlockIterator() { 228 return new BlockIterator(blockList); 229 230 } 231 232 /** 233 * Move block to the head of the list of blocks belonging to the data-node. 234 * @return the index of the head of the blockList 235 */ 236 int moveBlockToHead(BlockInfo b, int curIndex, int headIndex) { 237 blockList = b.moveBlockToHead(blockList, this, curIndex, headIndex); 238 return curIndex; 239 } 240 241 /** 242 * Used for testing only 243 * @return the head of the blockList 244 */ 245 @VisibleForTesting 246 BlockInfo getBlockListHeadForTesting(){ 247 return blockList; 248 } 249 250 void updateState(StorageReport r) { 251 capacity = r.getCapacity(); 252 dfsUsed = r.getDfsUsed(); 253 remaining = r.getRemaining(); 254 blockPoolUsed = r.getBlockPoolUsed(); 255 } 256 257 public DatanodeDescriptor getDatanodeDescriptor() { 258 return dn; 259 } 260 261 /** Increment the number of blocks scheduled for each given storage */ 262 public static void incrementBlocksScheduled(DatanodeStorageInfo... storages) { 263 for (DatanodeStorageInfo s : storages) { 264 s.getDatanodeDescriptor().incrementBlocksScheduled(); 265 } 266 } 267 268 @Override 269 public boolean equals(Object obj) { 270 if (this == obj) { 271 return true; 272 } else if (obj == null || !(obj instanceof DatanodeStorageInfo)) { 273 return false; 274 } 275 final DatanodeStorageInfo that = (DatanodeStorageInfo)obj; 276 return this.storageID.equals(that.storageID); 277 } 278 279 @Override 280 public int hashCode() { 281 return storageID.hashCode(); 282 } 283 284 @Override 285 public String toString() { 286 return "[" + storageType + "]" + storageID + ":" + state; 287 } 288}