001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs.server.datanode;
019
020import java.io.File;
021import java.io.FileInputStream;
022import java.io.FileOutputStream;
023import java.io.IOException;
024import java.util.ArrayList;
025import java.util.HashMap;
026import java.util.List;
027import java.util.Map;
028
029import org.apache.hadoop.classification.InterfaceAudience;
030import org.apache.hadoop.fs.FileUtil;
031import org.apache.hadoop.fs.HardLink;
032import org.apache.hadoop.hdfs.protocol.Block;
033import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
034import org.apache.hadoop.io.IOUtils;
035
036import com.google.common.annotations.VisibleForTesting;
037
038/**
039 * This class is used by datanodes to maintain meta data of its replicas.
040 * It provides a general interface for meta information of a replica.
041 */
042@InterfaceAudience.Private
043abstract public class ReplicaInfo extends Block implements Replica {
044  
045  /** volume where the replica belongs */
046  private FsVolumeSpi volume;
047  
048  /** directory where block & meta files belong */
049  
050  /**
051   * Base directory containing numerically-identified sub directories and
052   * possibly blocks.
053   */
054  private File baseDir;
055  
056  /**
057   * Ints representing the sub directory path from base dir to the directory
058   * containing this replica.
059   */
060  private int[] subDirs;
061  
062  private static final Map<String, File> internedBaseDirs = new HashMap<String, File>();
063
064  /**
065   * Constructor for a zero length replica
066   * @param blockId block id
067   * @param genStamp replica generation stamp
068   * @param vol volume where replica is located
069   * @param dir directory path where block and meta files are located
070   */
071  ReplicaInfo(long blockId, long genStamp, FsVolumeSpi vol, File dir) {
072    this( blockId, 0L, genStamp, vol, dir);
073  }
074  
075  /**
076   * Constructor
077   * @param block a block
078   * @param vol volume where replica is located
079   * @param dir directory path where block and meta files are located
080   */
081  ReplicaInfo(Block block, FsVolumeSpi vol, File dir) {
082    this(block.getBlockId(), block.getNumBytes(), 
083        block.getGenerationStamp(), vol, dir);
084  }
085  
086  /**
087   * Constructor
088   * @param blockId block id
089   * @param len replica length
090   * @param genStamp replica generation stamp
091   * @param vol volume where replica is located
092   * @param dir directory path where block and meta files are located
093   */
094  ReplicaInfo(long blockId, long len, long genStamp,
095      FsVolumeSpi vol, File dir) {
096    super(blockId, len, genStamp);
097    this.volume = vol;
098    setDirInternal(dir);
099  }
100
101  /**
102   * Copy constructor.
103   * @param from
104   */
105  ReplicaInfo(ReplicaInfo from) {
106    this(from, from.getVolume(), from.getDir());
107  }
108  
109  /**
110   * Get the full path of this replica's data file
111   * @return the full path of this replica's data file
112   */
113  public File getBlockFile() {
114    return new File(getDir(), getBlockName());
115  }
116  
117  /**
118   * Get the full path of this replica's meta file
119   * @return the full path of this replica's meta file
120   */
121  public File getMetaFile() {
122    return new File(getDir(),
123        DatanodeUtil.getMetaName(getBlockName(), getGenerationStamp()));
124  }
125  
126  /**
127   * Get the volume where this replica is located on disk
128   * @return the volume where this replica is located on disk
129   */
130  public FsVolumeSpi getVolume() {
131    return volume;
132  }
133  
134  /**
135   * Set the volume where this replica is located on disk
136   */
137  void setVolume(FsVolumeSpi vol) {
138    this.volume = vol;
139  }
140
141  /**
142   * Get the storageUuid of the volume that stores this replica.
143   */
144  @Override
145  public String getStorageUuid() {
146    return volume.getStorageID();
147  }
148  
149  /**
150   * Return the parent directory path where this replica is located
151   * @return the parent directory path where this replica is located
152   */
153  File getDir() {
154    if (subDirs == null) {
155      return null;
156    }
157
158    StringBuilder sb = new StringBuilder();
159    for (int i : subDirs) {
160      sb.append(DataStorage.BLOCK_SUBDIR_PREFIX);
161      sb.append(i);
162      sb.append("/");
163    }
164    File ret = new File(baseDir, sb.toString());
165    return ret;
166  }
167
168  /**
169   * Set the parent directory where this replica is located
170   * @param dir the parent directory where the replica is located
171   */
172  public void setDir(File dir) {
173    setDirInternal(dir);
174  }
175
176  private void setDirInternal(File dir) {
177    if (dir == null) {
178      subDirs = null;
179      baseDir = null;
180      return;
181    }
182
183    ReplicaDirInfo replicaDirInfo = parseSubDirs(dir);
184    this.subDirs = replicaDirInfo.subDirs;
185    
186    synchronized (internedBaseDirs) {
187      if (!internedBaseDirs.containsKey(replicaDirInfo.baseDirPath)) {
188        // Create a new String path of this file and make a brand new File object
189        // to guarantee we drop the reference to the underlying char[] storage.
190        File baseDir = new File(new String(replicaDirInfo.baseDirPath));
191        internedBaseDirs.put(replicaDirInfo.baseDirPath, baseDir);
192      }
193      this.baseDir = internedBaseDirs.get(replicaDirInfo.baseDirPath);
194    }
195  }
196  
197  @VisibleForTesting
198  public static class ReplicaDirInfo {
199    @VisibleForTesting
200    public String baseDirPath;
201    
202    @VisibleForTesting
203    public int[] subDirs;
204  }
205  
206  @VisibleForTesting
207  public static ReplicaDirInfo parseSubDirs(File dir) {
208    ReplicaDirInfo ret = new ReplicaDirInfo();
209    
210    File currentDir = dir;
211    List<Integer> subDirList = new ArrayList<Integer>();
212    while (currentDir.getName().startsWith(DataStorage.BLOCK_SUBDIR_PREFIX)) {
213      // Prepend the integer into the list.
214      subDirList.add(0, Integer.parseInt(currentDir.getName().replaceFirst(
215          DataStorage.BLOCK_SUBDIR_PREFIX, "")));
216      currentDir = currentDir.getParentFile();
217    }
218    ret.subDirs = new int[subDirList.size()];
219    for (int i = 0; i < subDirList.size(); i++) {
220      ret.subDirs[i] = subDirList.get(i);
221    }
222    
223    ret.baseDirPath = currentDir.getAbsolutePath();
224    
225    return ret;
226  }
227
228  /**
229   * check if this replica has already been unlinked.
230   * @return true if the replica has already been unlinked 
231   *         or no need to be detached; false otherwise
232   */
233  public boolean isUnlinked() {
234    return true;                // no need to be unlinked
235  }
236
237  /**
238   * set that this replica is unlinked
239   */
240  public void setUnlinked() {
241    // no need to be unlinked
242  }
243  
244   /**
245   * Copy specified file into a temporary file. Then rename the
246   * temporary file to the original name. This will cause any
247   * hardlinks to the original file to be removed. The temporary
248   * files are created in the same directory. The temporary files will
249   * be recovered (especially on Windows) on datanode restart.
250   */
251  private void unlinkFile(File file, Block b) throws IOException {
252    File tmpFile = DatanodeUtil.createTmpFile(b, DatanodeUtil.getUnlinkTmpFile(file));
253    try {
254      FileInputStream in = new FileInputStream(file);
255      try {
256        FileOutputStream out = new FileOutputStream(tmpFile);
257        try {
258          IOUtils.copyBytes(in, out, 16*1024);
259        } finally {
260          out.close();
261        }
262      } finally {
263        in.close();
264      }
265      if (file.length() != tmpFile.length()) {
266        throw new IOException("Copy of file " + file + " size " + file.length()+
267                              " into file " + tmpFile +
268                              " resulted in a size of " + tmpFile.length());
269      }
270      FileUtil.replaceFile(tmpFile, file);
271    } catch (IOException e) {
272      boolean done = tmpFile.delete();
273      if (!done) {
274        DataNode.LOG.info("detachFile failed to delete temporary file " +
275                          tmpFile);
276      }
277      throw e;
278    }
279  }
280
281  /**
282   * Remove a hard link by copying the block to a temporary place and 
283   * then moving it back
284   * @param numLinks number of hard links
285   * @return true if copy is successful; 
286   *         false if it is already detached or no need to be detached
287   * @throws IOException if there is any copy error
288   */
289  public boolean unlinkBlock(int numLinks) throws IOException {
290    if (isUnlinked()) {
291      return false;
292    }
293    File file = getBlockFile();
294    if (file == null || getVolume() == null) {
295      throw new IOException("detachBlock:Block not found. " + this);
296    }
297    File meta = getMetaFile();
298
299    if (HardLink.getLinkCount(file) > numLinks) {
300      DataNode.LOG.info("CopyOnWrite for block " + this);
301      unlinkFile(file, this);
302    }
303    if (HardLink.getLinkCount(meta) > numLinks) {
304      unlinkFile(meta, this);
305    }
306    setUnlinked();
307    return true;
308  }
309
310  /**
311   * Set this replica's generation stamp to be a newer one
312   * @param newGS new generation stamp
313   * @throws IOException is the new generation stamp is not greater than the current one
314   */
315  void setNewerGenerationStamp(long newGS) throws IOException {
316    long curGS = getGenerationStamp();
317    if (newGS <= curGS) {
318      throw new IOException("New generation stamp (" + newGS 
319          + ") must be greater than current one (" + curGS + ")");
320    }
321    setGenerationStamp(newGS);
322  }
323  
324  @Override  //Object
325  public String toString() {
326    return getClass().getSimpleName()
327        + ", " + super.toString()
328        + ", " + getState()
329        + "\n  getNumBytes()     = " + getNumBytes()
330        + "\n  getBytesOnDisk()  = " + getBytesOnDisk()
331        + "\n  getVisibleLength()= " + getVisibleLength()
332        + "\n  getVolume()       = " + getVolume()
333        + "\n  getBlockFile()    = " + getBlockFile();
334  }
335}