001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hdfs.server.namenode; 019 020import java.io.File; 021import java.io.FileInputStream; 022import java.io.FileOutputStream; 023import java.io.IOException; 024import java.io.InputStream; 025import java.net.HttpURLConnection; 026import java.net.InetSocketAddress; 027import java.net.URL; 028import java.security.DigestInputStream; 029import java.security.MessageDigest; 030import java.util.ArrayList; 031import java.util.List; 032 033import javax.servlet.ServletOutputStream; 034import javax.servlet.ServletResponse; 035import javax.servlet.http.HttpServletResponse; 036 037import org.apache.commons.logging.Log; 038import org.apache.commons.logging.LogFactory; 039import org.apache.hadoop.classification.InterfaceAudience; 040import org.apache.hadoop.conf.Configuration; 041import org.apache.hadoop.fs.FileUtil; 042import org.apache.hadoop.http.HttpConfig; 043import org.apache.hadoop.security.UserGroupInformation; 044import org.apache.hadoop.security.authentication.client.AuthenticationException; 045import org.apache.hadoop.util.Time; 046import org.apache.hadoop.hdfs.DFSConfigKeys; 047import org.apache.hadoop.hdfs.HdfsConfiguration; 048import org.apache.hadoop.hdfs.protocol.HdfsConstants; 049import org.apache.hadoop.hdfs.server.common.Storage; 050import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; 051import org.apache.hadoop.hdfs.server.common.StorageErrorReporter; 052import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; 053import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; 054import org.apache.hadoop.hdfs.util.DataTransferThrottler; 055import org.apache.hadoop.hdfs.web.URLConnectionFactory; 056import org.apache.hadoop.io.MD5Hash; 057import org.apache.hadoop.security.SecurityUtil; 058import org.apache.hadoop.util.Time; 059 060import com.google.common.annotations.VisibleForTesting; 061import com.google.common.collect.Lists; 062 063 064/** 065 * This class provides fetching a specified file from the NameNode. 066 */ 067@InterfaceAudience.Private 068public class TransferFsImage { 069 070 public final static String CONTENT_LENGTH = "Content-Length"; 071 public final static String MD5_HEADER = "X-MD5-Digest"; 072 @VisibleForTesting 073 static int timeout = 0; 074 private static URLConnectionFactory connectionFactory; 075 private static boolean isSpnegoEnabled; 076 077 static { 078 Configuration conf = new Configuration(); 079 connectionFactory = URLConnectionFactory 080 .newDefaultURLConnectionFactory(conf); 081 isSpnegoEnabled = UserGroupInformation.isSecurityEnabled(); 082 } 083 084 private static final Log LOG = LogFactory.getLog(TransferFsImage.class); 085 086 public static void downloadMostRecentImageToDirectory(URL infoServer, 087 File dir) throws IOException { 088 String fileId = GetImageServlet.getParamStringForMostRecentImage(); 089 getFileClient(infoServer, fileId, Lists.newArrayList(dir), 090 null, false); 091 } 092 093 public static MD5Hash downloadImageToStorage( 094 URL fsName, long imageTxId, Storage dstStorage, boolean needDigest) 095 throws IOException { 096 String fileid = GetImageServlet.getParamStringForImage( 097 imageTxId, dstStorage); 098 String fileName = NNStorage.getCheckpointImageFileName(imageTxId); 099 100 List<File> dstFiles = dstStorage.getFiles( 101 NameNodeDirType.IMAGE, fileName); 102 if (dstFiles.isEmpty()) { 103 throw new IOException("No targets in destination storage!"); 104 } 105 106 MD5Hash hash = getFileClient(fsName, fileid, dstFiles, dstStorage, needDigest); 107 LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size " + 108 dstFiles.get(0).length() + " bytes."); 109 return hash; 110 } 111 112 static void downloadEditsToStorage(URL fsName, RemoteEditLog log, 113 NNStorage dstStorage) throws IOException { 114 assert log.getStartTxId() > 0 && log.getEndTxId() > 0 : 115 "bad log: " + log; 116 String fileid = GetImageServlet.getParamStringForLog( 117 log, dstStorage); 118 String finalFileName = NNStorage.getFinalizedEditsFileName( 119 log.getStartTxId(), log.getEndTxId()); 120 121 List<File> finalFiles = dstStorage.getFiles(NameNodeDirType.EDITS, 122 finalFileName); 123 assert !finalFiles.isEmpty() : "No checkpoint targets."; 124 125 for (File f : finalFiles) { 126 if (f.exists() && FileUtil.canRead(f)) { 127 LOG.info("Skipping download of remote edit log " + 128 log + " since it already is stored locally at " + f); 129 return; 130 } else if (LOG.isDebugEnabled()) { 131 LOG.debug("Dest file: " + f); 132 } 133 } 134 135 final long milliTime = System.currentTimeMillis(); 136 String tmpFileName = NNStorage.getTemporaryEditsFileName( 137 log.getStartTxId(), log.getEndTxId(), milliTime); 138 List<File> tmpFiles = dstStorage.getFiles(NameNodeDirType.EDITS, 139 tmpFileName); 140 getFileClient(fsName, fileid, tmpFiles, dstStorage, false); 141 LOG.info("Downloaded file " + tmpFiles.get(0).getName() + " size " + 142 finalFiles.get(0).length() + " bytes."); 143 144 CheckpointFaultInjector.getInstance().beforeEditsRename(); 145 146 for (StorageDirectory sd : dstStorage.dirIterable(NameNodeDirType.EDITS)) { 147 File tmpFile = NNStorage.getTemporaryEditsFile(sd, 148 log.getStartTxId(), log.getEndTxId(), milliTime); 149 File finalizedFile = NNStorage.getFinalizedEditsFile(sd, 150 log.getStartTxId(), log.getEndTxId()); 151 if (LOG.isDebugEnabled()) { 152 LOG.debug("Renaming " + tmpFile + " to " + finalizedFile); 153 } 154 boolean success = tmpFile.renameTo(finalizedFile); 155 if (!success) { 156 LOG.warn("Unable to rename edits file from " + tmpFile 157 + " to " + finalizedFile); 158 } 159 } 160 } 161 162 /** 163 * Requests that the NameNode download an image from this node. 164 * 165 * @param fsName the http address for the remote NN 166 * @param myNNAddress the host/port where the local node is running an 167 * HTTPServer hosting GetImageServlet 168 * @param storage the storage directory to transfer the image from 169 * @param txid the transaction ID of the image to be uploaded 170 */ 171 public static void uploadImageFromStorage(URL fsName, 172 URL myNNAddress, 173 Storage storage, long txid) throws IOException { 174 175 String fileid = GetImageServlet.getParamStringToPutImage( 176 txid, myNNAddress, storage); 177 // this doesn't directly upload an image, but rather asks the NN 178 // to connect back to the 2NN to download the specified image. 179 try { 180 TransferFsImage.getFileClient(fsName, fileid, null, null, false); 181 } catch (HttpGetFailedException e) { 182 if (e.getResponseCode() == HttpServletResponse.SC_CONFLICT) { 183 // this is OK - this means that a previous attempt to upload 184 // this checkpoint succeeded even though we thought it failed. 185 LOG.info("Image upload with txid " + txid + 186 " conflicted with a previous image upload to the " + 187 "same NameNode. Continuing...", e); 188 return; 189 } else { 190 throw e; 191 } 192 } 193 LOG.info("Uploaded image with txid " + txid + " to namenode at " + 194 fsName); 195 } 196 197 198 /** 199 * A server-side method to respond to a getfile http request 200 * Copies the contents of the local file into the output stream. 201 */ 202 public static void getFileServer(ServletResponse response, File localfile, 203 FileInputStream infile, 204 DataTransferThrottler throttler) 205 throws IOException { 206 byte buf[] = new byte[HdfsConstants.IO_FILE_BUFFER_SIZE]; 207 ServletOutputStream out = null; 208 try { 209 CheckpointFaultInjector.getInstance() 210 .aboutToSendFile(localfile); 211 out = response.getOutputStream(); 212 213 if (CheckpointFaultInjector.getInstance(). 214 shouldSendShortFile(localfile)) { 215 // Test sending image shorter than localfile 216 long len = localfile.length(); 217 buf = new byte[(int)Math.min(len/2, HdfsConstants.IO_FILE_BUFFER_SIZE)]; 218 // This will read at most half of the image 219 // and the rest of the image will be sent over the wire 220 infile.read(buf); 221 } 222 int num = 1; 223 while (num > 0) { 224 num = infile.read(buf); 225 if (num <= 0) { 226 break; 227 } 228 if (CheckpointFaultInjector.getInstance() 229 .shouldCorruptAByte(localfile)) { 230 // Simulate a corrupted byte on the wire 231 LOG.warn("SIMULATING A CORRUPT BYTE IN IMAGE TRANSFER!"); 232 buf[0]++; 233 } 234 235 out.write(buf, 0, num); 236 if (throttler != null) { 237 throttler.throttle(num); 238 } 239 } 240 } finally { 241 if (out != null) { 242 out.close(); 243 } 244 } 245 } 246 247 /** 248 * Client-side Method to fetch file from a server 249 * Copies the response from the URL to a list of local files. 250 * @param dstStorage if an error occurs writing to one of the files, 251 * this storage object will be notified. 252 * @Return a digest of the received file if getChecksum is true 253 */ 254 static MD5Hash getFileClient(URL infoServer, 255 String queryString, List<File> localPaths, 256 Storage dstStorage, boolean getChecksum) throws IOException { 257 URL url = new URL(infoServer, "/getimage?" + queryString); 258 LOG.info("Opening connection to " + url); 259 return doGetUrl(url, localPaths, dstStorage, getChecksum); 260 } 261 262 public static MD5Hash doGetUrl(URL url, List<File> localPaths, 263 Storage dstStorage, boolean getChecksum) throws IOException { 264 long startTime = Time.monotonicNow(); 265 HttpURLConnection connection; 266 try { 267 connection = (HttpURLConnection) 268 connectionFactory.openConnection(url, isSpnegoEnabled); 269 } catch (AuthenticationException e) { 270 throw new IOException(e); 271 } 272 273 if (timeout <= 0) { 274 Configuration conf = new HdfsConfiguration(); 275 timeout = conf.getInt(DFSConfigKeys.DFS_IMAGE_TRANSFER_TIMEOUT_KEY, 276 DFSConfigKeys.DFS_IMAGE_TRANSFER_TIMEOUT_DEFAULT); 277 } 278 279 if (timeout > 0) { 280 connection.setConnectTimeout(timeout); 281 connection.setReadTimeout(timeout); 282 } 283 284 if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) { 285 throw new HttpGetFailedException( 286 "Image transfer servlet at " + url + 287 " failed with status code " + connection.getResponseCode() + 288 "\nResponse message:\n" + connection.getResponseMessage(), 289 connection); 290 } 291 292 long advertisedSize; 293 String contentLength = connection.getHeaderField(CONTENT_LENGTH); 294 if (contentLength != null) { 295 advertisedSize = Long.parseLong(contentLength); 296 } else { 297 throw new IOException(CONTENT_LENGTH + " header is not provided " + 298 "by the namenode when trying to fetch " + url); 299 } 300 301 if (localPaths != null) { 302 String fsImageName = connection.getHeaderField( 303 GetImageServlet.HADOOP_IMAGE_EDITS_HEADER); 304 // If the local paths refer to directories, use the server-provided header 305 // as the filename within that directory 306 List<File> newLocalPaths = new ArrayList<File>(); 307 for (File localPath : localPaths) { 308 if (localPath.isDirectory()) { 309 if (fsImageName == null) { 310 throw new IOException("No filename header provided by server"); 311 } 312 newLocalPaths.add(new File(localPath, fsImageName)); 313 } else { 314 newLocalPaths.add(localPath); 315 } 316 } 317 localPaths = newLocalPaths; 318 } 319 320 MD5Hash advertisedDigest = parseMD5Header(connection); 321 322 long received = 0; 323 InputStream stream = connection.getInputStream(); 324 MessageDigest digester = null; 325 if (getChecksum) { 326 digester = MD5Hash.getDigester(); 327 stream = new DigestInputStream(stream, digester); 328 } 329 boolean finishedReceiving = false; 330 331 List<FileOutputStream> outputStreams = Lists.newArrayList(); 332 333 try { 334 if (localPaths != null) { 335 for (File f : localPaths) { 336 try { 337 if (f.exists()) { 338 LOG.warn("Overwriting existing file " + f 339 + " with file downloaded from " + url); 340 } 341 outputStreams.add(new FileOutputStream(f)); 342 } catch (IOException ioe) { 343 LOG.warn("Unable to download file " + f, ioe); 344 // This will be null if we're downloading the fsimage to a file 345 // outside of an NNStorage directory. 346 if (dstStorage != null && 347 (dstStorage instanceof StorageErrorReporter)) { 348 ((StorageErrorReporter)dstStorage).reportErrorOnFile(f); 349 } 350 } 351 } 352 353 if (outputStreams.isEmpty()) { 354 throw new IOException( 355 "Unable to download to any storage directory"); 356 } 357 } 358 359 int num = 1; 360 byte[] buf = new byte[HdfsConstants.IO_FILE_BUFFER_SIZE]; 361 while (num > 0) { 362 num = stream.read(buf); 363 if (num > 0) { 364 received += num; 365 for (FileOutputStream fos : outputStreams) { 366 fos.write(buf, 0, num); 367 } 368 } 369 } 370 finishedReceiving = true; 371 } finally { 372 stream.close(); 373 for (FileOutputStream fos : outputStreams) { 374 fos.getChannel().force(true); 375 fos.close(); 376 } 377 if (finishedReceiving && received != advertisedSize) { 378 // only throw this exception if we think we read all of it on our end 379 // -- otherwise a client-side IOException would be masked by this 380 // exception that makes it look like a server-side problem! 381 throw new IOException("File " + url + " received length " + received + 382 " is not of the advertised size " + 383 advertisedSize); 384 } 385 } 386 double xferSec = Math.max( 387 ((float)(Time.monotonicNow() - startTime)) / 1000.0, 0.001); 388 long xferKb = received / 1024; 389 LOG.info(String.format("Transfer took %.2fs at %.2f KB/s", 390 xferSec, xferKb / xferSec)); 391 392 if (digester != null) { 393 MD5Hash computedDigest = new MD5Hash(digester.digest()); 394 395 if (advertisedDigest != null && 396 !computedDigest.equals(advertisedDigest)) { 397 throw new IOException("File " + url + " computed digest " + 398 computedDigest + " does not match advertised digest " + 399 advertisedDigest); 400 } 401 return computedDigest; 402 } else { 403 return null; 404 } 405 } 406 407 private static MD5Hash parseMD5Header(HttpURLConnection connection) { 408 String header = connection.getHeaderField(MD5_HEADER); 409 return (header != null) ? new MD5Hash(header) : null; 410 } 411 412 public static class HttpGetFailedException extends IOException { 413 private static final long serialVersionUID = 1L; 414 private final int responseCode; 415 416 HttpGetFailedException(String msg, HttpURLConnection connection) throws IOException { 417 super(msg); 418 this.responseCode = connection.getResponseCode(); 419 } 420 421 public int getResponseCode() { 422 return responseCode; 423 } 424 } 425 426}