001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs.server.blockmanagement;
019
020import org.apache.hadoop.classification.InterfaceAudience;
021import org.apache.hadoop.hdfs.protocol.Block;
022import org.apache.hadoop.hdfs.server.namenode.NameNode;
023import org.apache.hadoop.ipc.Server;
024
025import java.util.*;
026
027/**
028 * Stores information about all corrupt blocks in the File System.
029 * A Block is considered corrupt only if all of its replicas are
030 * corrupt. While reporting replicas of a Block, we hide any corrupt
031 * copies. These copies are removed once Block is found to have 
032 * expected number of good replicas.
033 * Mapping: Block -> TreeSet<DatanodeDescriptor> 
034 */
035
036@InterfaceAudience.Private
037public class CorruptReplicasMap{
038
039  /** The corruption reason code */
040  public static enum Reason {
041    NONE,                // not specified.
042    ANY,                 // wildcard reason
043    GENSTAMP_MISMATCH,   // mismatch in generation stamps
044    SIZE_MISMATCH,       // mismatch in sizes
045    INVALID_STATE,       // invalid state
046    CORRUPTION_REPORTED  // client or datanode reported the corruption
047  }
048
049  private SortedMap<Block, Map<DatanodeDescriptor, Reason>> corruptReplicasMap =
050    new TreeMap<Block, Map<DatanodeDescriptor, Reason>>();
051  
052  /**
053   * Mark the block belonging to datanode as corrupt.
054   *
055   * @param blk Block to be added to CorruptReplicasMap
056   * @param dn DatanodeDescriptor which holds the corrupt replica
057   * @param reason a textual reason (for logging purposes)
058   */
059  public void addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn,
060      String reason) {
061    addToCorruptReplicasMap(blk, dn, reason, Reason.NONE);
062  }
063
064  /**
065   * Mark the block belonging to datanode as corrupt.
066   *
067   * @param blk Block to be added to CorruptReplicasMap
068   * @param dn DatanodeDescriptor which holds the corrupt replica
069   * @param reason a textual reason (for logging purposes)
070   * @param reasonCode the enum representation of the reason
071   */
072  public void addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn,
073      String reason, Reason reasonCode) {
074    Map <DatanodeDescriptor, Reason> nodes = corruptReplicasMap.get(blk);
075    if (nodes == null) {
076      nodes = new HashMap<DatanodeDescriptor, Reason>();
077      corruptReplicasMap.put(blk, nodes);
078    }
079    
080    String reasonText;
081    if (reason != null) {
082      reasonText = " because " + reason;
083    } else {
084      reasonText = "";
085    }
086    
087    if (!nodes.keySet().contains(dn)) {
088      NameNode.blockStateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+
089                                   blk.getBlockName() +
090                                   " added as corrupt on " + dn +
091                                   " by " + Server.getRemoteIp() +
092                                   reasonText);
093    } else {
094      NameNode.blockStateChangeLog.info("BLOCK NameSystem.addToCorruptReplicasMap: "+
095                                   "duplicate requested for " + 
096                                   blk.getBlockName() + " to add as corrupt " +
097                                   "on " + dn +
098                                   " by " + Server.getRemoteIp() +
099                                   reasonText);
100    }
101    // Add the node or update the reason.
102    nodes.put(dn, reasonCode);
103  }
104
105  /**
106   * Remove Block from CorruptBlocksMap
107   *
108   * @param blk Block to be removed
109   */
110  void removeFromCorruptReplicasMap(Block blk) {
111    if (corruptReplicasMap != null) {
112      corruptReplicasMap.remove(blk);
113    }
114  }
115
116  /**
117   * Remove the block at the given datanode from CorruptBlockMap
118   * @param blk block to be removed
119   * @param datanode datanode where the block is located
120   * @return true if the removal is successful; 
121             false if the replica is not in the map
122   */ 
123  boolean removeFromCorruptReplicasMap(Block blk, DatanodeDescriptor datanode) {
124    return removeFromCorruptReplicasMap(blk, datanode, Reason.ANY);
125  }
126
127  boolean removeFromCorruptReplicasMap(Block blk, DatanodeDescriptor datanode,
128      Reason reason) {
129    Map <DatanodeDescriptor, Reason> datanodes = corruptReplicasMap.get(blk);
130    boolean removed = false;
131    if (datanodes==null)
132      return false;
133
134    // if reasons can be compared but don't match, return false.
135    Reason storedReason = datanodes.get(datanode);
136    if (reason != Reason.ANY && storedReason != null &&
137        reason != storedReason) {
138      return false;
139    }
140
141    if (datanodes.remove(datanode) != null) { // remove the replicas
142      if (datanodes.isEmpty()) {
143        // remove the block if there is no more corrupted replicas
144        corruptReplicasMap.remove(blk);
145      }
146      return true;
147    }
148    return false;
149  }
150    
151
152  /**
153   * Get Nodes which have corrupt replicas of Block
154   * 
155   * @param blk Block for which nodes are requested
156   * @return collection of nodes. Null if does not exists
157   */
158  Collection<DatanodeDescriptor> getNodes(Block blk) {
159    Map <DatanodeDescriptor, Reason> nodes = corruptReplicasMap.get(blk);
160    if (nodes == null)
161      return null;
162    return nodes.keySet();
163  }
164
165  /**
166   * Check if replica belonging to Datanode is corrupt
167   *
168   * @param blk Block to check
169   * @param node DatanodeDescriptor which holds the replica
170   * @return true if replica is corrupt, false if does not exists in this map
171   */
172  boolean isReplicaCorrupt(Block blk, DatanodeDescriptor node) {
173    Collection<DatanodeDescriptor> nodes = getNodes(blk);
174    return ((nodes != null) && (nodes.contains(node)));
175  }
176
177  public int numCorruptReplicas(Block blk) {
178    Collection<DatanodeDescriptor> nodes = getNodes(blk);
179    return (nodes == null) ? 0 : nodes.size();
180  }
181  
182  public int size() {
183    return corruptReplicasMap.size();
184  }
185
186  /**
187   * Return a range of corrupt replica block ids. Up to numExpectedBlocks 
188   * blocks starting at the next block after startingBlockId are returned
189   * (fewer if numExpectedBlocks blocks are unavailable). If startingBlockId 
190   * is null, up to numExpectedBlocks blocks are returned from the beginning.
191   * If startingBlockId cannot be found, null is returned.
192   *
193   * @param numExpectedBlocks Number of block ids to return.
194   *  0 <= numExpectedBlocks <= 100
195   * @param startingBlockId Block id from which to start. If null, start at
196   *  beginning.
197   * @return Up to numExpectedBlocks blocks from startingBlockId if it exists
198   *
199   */
200  long[] getCorruptReplicaBlockIds(int numExpectedBlocks,
201                                   Long startingBlockId) {
202    if (numExpectedBlocks < 0 || numExpectedBlocks > 100) {
203      return null;
204    }
205    
206    Iterator<Block> blockIt = corruptReplicasMap.keySet().iterator();
207    
208    // if the starting block id was specified, iterate over keys until
209    // we find the matching block. If we find a matching block, break
210    // to leave the iterator on the next block after the specified block. 
211    if (startingBlockId != null) {
212      boolean isBlockFound = false;
213      while (blockIt.hasNext()) {
214        Block b = blockIt.next();
215        if (b.getBlockId() == startingBlockId) {
216          isBlockFound = true;
217          break; 
218        }
219      }
220      
221      if (!isBlockFound) {
222        return null;
223      }
224    }
225
226    ArrayList<Long> corruptReplicaBlockIds = new ArrayList<Long>();
227
228    // append up to numExpectedBlocks blockIds to our list
229    for(int i=0; i<numExpectedBlocks && blockIt.hasNext(); i++) {
230      corruptReplicaBlockIds.add(blockIt.next().getBlockId());
231    }
232    
233    long[] ret = new long[corruptReplicaBlockIds.size()];
234    for(int i=0; i<ret.length; i++) {
235      ret[i] = corruptReplicaBlockIds.get(i);
236    }
237    
238    return ret;
239  }  
240}