001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hdfs.server.blockmanagement; 019 020import org.apache.hadoop.classification.InterfaceAudience; 021import org.apache.hadoop.hdfs.protocol.Block; 022import org.apache.hadoop.hdfs.server.namenode.NameNode; 023import org.apache.hadoop.ipc.Server; 024 025import java.util.*; 026 027/** 028 * Stores information about all corrupt blocks in the File System. 029 * A Block is considered corrupt only if all of its replicas are 030 * corrupt. While reporting replicas of a Block, we hide any corrupt 031 * copies. These copies are removed once Block is found to have 032 * expected number of good replicas. 033 * Mapping: Block -> TreeSet<DatanodeDescriptor> 034 */ 035 036@InterfaceAudience.Private 037public class CorruptReplicasMap{ 038 039 /** The corruption reason code */ 040 public static enum Reason { 041 NONE, // not specified. 042 ANY, // wildcard reason 043 GENSTAMP_MISMATCH, // mismatch in generation stamps 044 SIZE_MISMATCH, // mismatch in sizes 045 INVALID_STATE, // invalid state 046 CORRUPTION_REPORTED // client or datanode reported the corruption 047 } 048 049 private SortedMap<Block, Map<DatanodeDescriptor, Reason>> corruptReplicasMap = 050 new TreeMap<Block, Map<DatanodeDescriptor, Reason>>(); 051 052 /** 053 * Mark the block belonging to datanode as corrupt. 054 * 055 * @param blk Block to be added to CorruptReplicasMap 056 * @param dn DatanodeDescriptor which holds the corrupt replica 057 * @param reason a textual reason (for logging purposes) 058 */ 059 public void addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn, 060 String reason) { 061 addToCorruptReplicasMap(blk, dn, reason, Reason.NONE); 062 } 063 064 /** 065 * Mark the block belonging to datanode as corrupt. 066 * 067 * @param blk Block to be added to CorruptReplicasMap 068 * @param dn DatanodeDescriptor which holds the corrupt replica 069 * @param reason a textual reason (for logging purposes) 070 * @param reasonCode the enum representation of the reason 071 */ 072 public void addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn, 073 String reason, Reason reasonCode) { 074 Map <DatanodeDescriptor, Reason> nodes = corruptReplicasMap.get(blk); 075 if (nodes == null) { 076 nodes = new HashMap<DatanodeDescriptor, Reason>(); 077 corruptReplicasMap.put(blk, nodes); 078 } 079 080 String reasonText; 081 if (reason != null) { 082 reasonText = " because " + reason; 083 } else { 084 reasonText = ""; 085 } 086 087 if (!nodes.keySet().contains(dn)) { 088 NameNode.blockStateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+ 089 blk.getBlockName() + 090 " added as corrupt on " + dn + 091 " by " + Server.getRemoteIp() + 092 reasonText); 093 } else { 094 NameNode.blockStateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+ 095 "duplicate requested for " + 096 blk.getBlockName() + " to add as corrupt " + 097 "on " + dn + 098 " by " + Server.getRemoteIp() + 099 reasonText); 100 } 101 // Add the node or update the reason. 102 nodes.put(dn, reasonCode); 103 } 104 105 /** 106 * Remove Block from CorruptBlocksMap 107 * 108 * @param blk Block to be removed 109 */ 110 void removeFromCorruptReplicasMap(Block blk) { 111 if (corruptReplicasMap != null) { 112 corruptReplicasMap.remove(blk); 113 } 114 } 115 116 /** 117 * Remove the block at the given datanode from CorruptBlockMap 118 * @param blk block to be removed 119 * @param datanode datanode where the block is located 120 * @return true if the removal is successful; 121 false if the replica is not in the map 122 */ 123 boolean removeFromCorruptReplicasMap(Block blk, DatanodeDescriptor datanode) { 124 return removeFromCorruptReplicasMap(blk, datanode, Reason.ANY); 125 } 126 127 boolean removeFromCorruptReplicasMap(Block blk, DatanodeDescriptor datanode, 128 Reason reason) { 129 Map <DatanodeDescriptor, Reason> datanodes = corruptReplicasMap.get(blk); 130 boolean removed = false; 131 if (datanodes==null) 132 return false; 133 134 // if reasons can be compared but don't match, return false. 135 Reason storedReason = datanodes.get(datanode); 136 if (reason != Reason.ANY && storedReason != null && 137 reason != storedReason) { 138 return false; 139 } 140 141 if (datanodes.remove(datanode) != null) { // remove the replicas 142 if (datanodes.isEmpty()) { 143 // remove the block if there is no more corrupted replicas 144 corruptReplicasMap.remove(blk); 145 } 146 return true; 147 } 148 return false; 149 } 150 151 152 /** 153 * Get Nodes which have corrupt replicas of Block 154 * 155 * @param blk Block for which nodes are requested 156 * @return collection of nodes. Null if does not exists 157 */ 158 Collection<DatanodeDescriptor> getNodes(Block blk) { 159 Map <DatanodeDescriptor, Reason> nodes = corruptReplicasMap.get(blk); 160 if (nodes == null) 161 return null; 162 return nodes.keySet(); 163 } 164 165 /** 166 * Check if replica belonging to Datanode is corrupt 167 * 168 * @param blk Block to check 169 * @param node DatanodeDescriptor which holds the replica 170 * @return true if replica is corrupt, false if does not exists in this map 171 */ 172 boolean isReplicaCorrupt(Block blk, DatanodeDescriptor node) { 173 Collection<DatanodeDescriptor> nodes = getNodes(blk); 174 return ((nodes != null) && (nodes.contains(node))); 175 } 176 177 public int numCorruptReplicas(Block blk) { 178 Collection<DatanodeDescriptor> nodes = getNodes(blk); 179 return (nodes == null) ? 0 : nodes.size(); 180 } 181 182 public int size() { 183 return corruptReplicasMap.size(); 184 } 185 186 /** 187 * Return a range of corrupt replica block ids. Up to numExpectedBlocks 188 * blocks starting at the next block after startingBlockId are returned 189 * (fewer if numExpectedBlocks blocks are unavailable). If startingBlockId 190 * is null, up to numExpectedBlocks blocks are returned from the beginning. 191 * If startingBlockId cannot be found, null is returned. 192 * 193 * @param numExpectedBlocks Number of block ids to return. 194 * 0 <= numExpectedBlocks <= 100 195 * @param startingBlockId Block id from which to start. If null, start at 196 * beginning. 197 * @return Up to numExpectedBlocks blocks from startingBlockId if it exists 198 * 199 */ 200 long[] getCorruptReplicaBlockIds(int numExpectedBlocks, 201 Long startingBlockId) { 202 if (numExpectedBlocks < 0 || numExpectedBlocks > 100) { 203 return null; 204 } 205 206 Iterator<Block> blockIt = corruptReplicasMap.keySet().iterator(); 207 208 // if the starting block id was specified, iterate over keys until 209 // we find the matching block. If we find a matching block, break 210 // to leave the iterator on the next block after the specified block. 211 if (startingBlockId != null) { 212 boolean isBlockFound = false; 213 while (blockIt.hasNext()) { 214 Block b = blockIt.next(); 215 if (b.getBlockId() == startingBlockId) { 216 isBlockFound = true; 217 break; 218 } 219 } 220 221 if (!isBlockFound) { 222 return null; 223 } 224 } 225 226 ArrayList<Long> corruptReplicaBlockIds = new ArrayList<Long>(); 227 228 // append up to numExpectedBlocks blockIds to our list 229 for(int i=0; i<numExpectedBlocks && blockIt.hasNext(); i++) { 230 corruptReplicaBlockIds.add(blockIt.next().getBlockId()); 231 } 232 233 long[] ret = new long[corruptReplicaBlockIds.size()]; 234 for(int i=0; i<ret.length; i++) { 235 ret[i] = corruptReplicaBlockIds.get(i); 236 } 237 238 return ret; 239 } 240}