001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hdfs.server.datanode.web.resources; 019 020import java.io.IOException; 021import java.io.InputStream; 022import java.net.InetSocketAddress; 023import java.net.URI; 024import java.net.URISyntaxException; 025import java.security.PrivilegedExceptionAction; 026import java.util.EnumSet; 027 028import javax.servlet.ServletContext; 029import javax.servlet.http.HttpServletResponse; 030import javax.ws.rs.Consumes; 031import javax.ws.rs.DefaultValue; 032import javax.ws.rs.GET; 033import javax.ws.rs.POST; 034import javax.ws.rs.PUT; 035import javax.ws.rs.Path; 036import javax.ws.rs.PathParam; 037import javax.ws.rs.Produces; 038import javax.ws.rs.QueryParam; 039import javax.ws.rs.core.Context; 040import javax.ws.rs.core.MediaType; 041import javax.ws.rs.core.Response; 042 043import org.apache.commons.logging.Log; 044import org.apache.commons.logging.LogFactory; 045import org.apache.hadoop.conf.Configuration; 046import org.apache.hadoop.fs.CreateFlag; 047import org.apache.hadoop.fs.FSDataOutputStream; 048import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; 049import org.apache.hadoop.fs.permission.FsPermission; 050import org.apache.hadoop.hdfs.DFSClient; 051import org.apache.hadoop.hdfs.client.HdfsDataInputStream; 052import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; 053import org.apache.hadoop.hdfs.server.datanode.DataNode; 054import org.apache.hadoop.hdfs.server.namenode.NameNode; 055import org.apache.hadoop.hdfs.web.JsonUtil; 056import org.apache.hadoop.hdfs.web.ParamFilter; 057import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; 058import org.apache.hadoop.hdfs.web.resources.BlockSizeParam; 059import org.apache.hadoop.hdfs.web.resources.BufferSizeParam; 060import org.apache.hadoop.hdfs.web.resources.DelegationParam; 061import org.apache.hadoop.hdfs.web.resources.GetOpParam; 062import org.apache.hadoop.hdfs.web.resources.HttpOpParam; 063import org.apache.hadoop.hdfs.web.resources.LengthParam; 064import org.apache.hadoop.hdfs.web.resources.NamenodeRpcAddressParam; 065import org.apache.hadoop.hdfs.web.resources.OffsetParam; 066import org.apache.hadoop.hdfs.web.resources.OverwriteParam; 067import org.apache.hadoop.hdfs.web.resources.Param; 068import org.apache.hadoop.hdfs.web.resources.PermissionParam; 069import org.apache.hadoop.hdfs.web.resources.PostOpParam; 070import org.apache.hadoop.hdfs.web.resources.PutOpParam; 071import org.apache.hadoop.hdfs.web.resources.ReplicationParam; 072import org.apache.hadoop.hdfs.web.resources.UriFsPathParam; 073import org.apache.hadoop.io.IOUtils; 074import org.apache.hadoop.security.SecurityUtil; 075import org.apache.hadoop.security.UserGroupInformation; 076import org.apache.hadoop.security.token.Token; 077 078import com.sun.jersey.spi.container.ResourceFilters; 079 080/** Web-hdfs DataNode implementation. */ 081@Path("") 082@ResourceFilters(ParamFilter.class) 083public class DatanodeWebHdfsMethods { 084 public static final Log LOG = LogFactory.getLog(DatanodeWebHdfsMethods.class); 085 086 private static final UriFsPathParam ROOT = new UriFsPathParam(""); 087 088 private @Context ServletContext context; 089 private @Context HttpServletResponse response; 090 091 private void init(final UserGroupInformation ugi, 092 final DelegationParam delegation, final InetSocketAddress nnRpcAddr, 093 final UriFsPathParam path, final HttpOpParam<?> op, 094 final Param<?, ?>... parameters) throws IOException { 095 if (LOG.isTraceEnabled()) { 096 LOG.trace("HTTP " + op.getValue().getType() + ": " + op + ", " + path 097 + ", ugi=" + ugi + Param.toSortedString(", ", parameters)); 098 } 099 if (nnRpcAddr == null) { 100 throw new IllegalArgumentException(NamenodeRpcAddressParam.NAME 101 + " is not specified."); 102 } 103 104 //clear content type 105 response.setContentType(null); 106 107 if (UserGroupInformation.isSecurityEnabled()) { 108 //add a token for RPC. 109 final Token<DelegationTokenIdentifier> token = 110 new Token<DelegationTokenIdentifier>(); 111 token.decodeFromUrlString(delegation.getValue()); 112 SecurityUtil.setTokenService(token, nnRpcAddr); 113 token.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND); 114 ugi.addToken(token); 115 } 116 } 117 118 /** Handle HTTP PUT request for the root. */ 119 @PUT 120 @Path("/") 121 @Consumes({"*/*"}) 122 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) 123 public Response putRoot( 124 final InputStream in, 125 @Context final UserGroupInformation ugi, 126 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) 127 final DelegationParam delegation, 128 @QueryParam(NamenodeRpcAddressParam.NAME) 129 @DefaultValue(NamenodeRpcAddressParam.DEFAULT) 130 final NamenodeRpcAddressParam namenodeRpcAddress, 131 @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT) 132 final PutOpParam op, 133 @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT) 134 final PermissionParam permission, 135 @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT) 136 final OverwriteParam overwrite, 137 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) 138 final BufferSizeParam bufferSize, 139 @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT) 140 final ReplicationParam replication, 141 @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT) 142 final BlockSizeParam blockSize 143 ) throws IOException, InterruptedException { 144 return put(in, ugi, delegation, namenodeRpcAddress, ROOT, op, permission, 145 overwrite, bufferSize, replication, blockSize); 146 } 147 148 /** Handle HTTP PUT request. */ 149 @PUT 150 @Path("{" + UriFsPathParam.NAME + ":.*}") 151 @Consumes({"*/*"}) 152 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) 153 public Response put( 154 final InputStream in, 155 @Context final UserGroupInformation ugi, 156 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) 157 final DelegationParam delegation, 158 @QueryParam(NamenodeRpcAddressParam.NAME) 159 @DefaultValue(NamenodeRpcAddressParam.DEFAULT) 160 final NamenodeRpcAddressParam namenodeRpcAddress, 161 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path, 162 @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT) 163 final PutOpParam op, 164 @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT) 165 final PermissionParam permission, 166 @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT) 167 final OverwriteParam overwrite, 168 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) 169 final BufferSizeParam bufferSize, 170 @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT) 171 final ReplicationParam replication, 172 @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT) 173 final BlockSizeParam blockSize 174 ) throws IOException, InterruptedException { 175 176 final InetSocketAddress nnRpcAddr = namenodeRpcAddress.getValue(); 177 init(ugi, delegation, nnRpcAddr, path, op, permission, 178 overwrite, bufferSize, replication, blockSize); 179 180 return ugi.doAs(new PrivilegedExceptionAction<Response>() { 181 @Override 182 public Response run() throws IOException, URISyntaxException { 183 return put(in, ugi, delegation, nnRpcAddr, path.getAbsolutePath(), op, 184 permission, overwrite, bufferSize, replication, blockSize); 185 } 186 }); 187 } 188 189 private Response put( 190 final InputStream in, 191 final UserGroupInformation ugi, 192 final DelegationParam delegation, 193 final InetSocketAddress nnRpcAddr, 194 final String fullpath, 195 final PutOpParam op, 196 final PermissionParam permission, 197 final OverwriteParam overwrite, 198 final BufferSizeParam bufferSize, 199 final ReplicationParam replication, 200 final BlockSizeParam blockSize 201 ) throws IOException, URISyntaxException { 202 final DataNode datanode = (DataNode)context.getAttribute("datanode"); 203 204 switch(op.getValue()) { 205 case CREATE: 206 { 207 final Configuration conf = new Configuration(datanode.getConf()); 208 conf.set(FsPermission.UMASK_LABEL, "000"); 209 210 final int b = bufferSize.getValue(conf); 211 DFSClient dfsclient = new DFSClient(nnRpcAddr, conf); 212 FSDataOutputStream out = null; 213 try { 214 out = new FSDataOutputStream(dfsclient.create( 215 fullpath, permission.getFsPermission(), 216 overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE) 217 : EnumSet.of(CreateFlag.CREATE), 218 replication.getValue(conf), blockSize.getValue(conf), null, b, null), null); 219 IOUtils.copyBytes(in, out, b); 220 out.close(); 221 out = null; 222 dfsclient.close(); 223 dfsclient = null; 224 } finally { 225 IOUtils.cleanup(LOG, out); 226 IOUtils.cleanup(LOG, dfsclient); 227 } 228 final InetSocketAddress nnHttpAddr = NameNode.getHttpAddress(conf); 229 final URI uri = new URI(WebHdfsFileSystem.SCHEME, null, 230 nnHttpAddr.getHostName(), nnHttpAddr.getPort(), fullpath, null, null); 231 return Response.created(uri).type(MediaType.APPLICATION_OCTET_STREAM).build(); 232 } 233 default: 234 throw new UnsupportedOperationException(op + " is not supported"); 235 } 236 } 237 238 /** Handle HTTP POST request for the root for the root. */ 239 @POST 240 @Path("/") 241 @Consumes({"*/*"}) 242 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) 243 public Response postRoot( 244 final InputStream in, 245 @Context final UserGroupInformation ugi, 246 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) 247 final DelegationParam delegation, 248 @QueryParam(NamenodeRpcAddressParam.NAME) 249 @DefaultValue(NamenodeRpcAddressParam.DEFAULT) 250 final NamenodeRpcAddressParam namenodeRpcAddress, 251 @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT) 252 final PostOpParam op, 253 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) 254 final BufferSizeParam bufferSize 255 ) throws IOException, InterruptedException { 256 return post(in, ugi, delegation, namenodeRpcAddress, ROOT, op, bufferSize); 257 } 258 259 /** Handle HTTP POST request. */ 260 @POST 261 @Path("{" + UriFsPathParam.NAME + ":.*}") 262 @Consumes({"*/*"}) 263 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) 264 public Response post( 265 final InputStream in, 266 @Context final UserGroupInformation ugi, 267 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) 268 final DelegationParam delegation, 269 @QueryParam(NamenodeRpcAddressParam.NAME) 270 @DefaultValue(NamenodeRpcAddressParam.DEFAULT) 271 final NamenodeRpcAddressParam namenodeRpcAddress, 272 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path, 273 @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT) 274 final PostOpParam op, 275 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) 276 final BufferSizeParam bufferSize 277 ) throws IOException, InterruptedException { 278 279 final InetSocketAddress nnRpcAddr = namenodeRpcAddress.getValue(); 280 init(ugi, delegation, nnRpcAddr, path, op, bufferSize); 281 282 return ugi.doAs(new PrivilegedExceptionAction<Response>() { 283 @Override 284 public Response run() throws IOException { 285 return post(in, ugi, delegation, nnRpcAddr, path.getAbsolutePath(), op, 286 bufferSize); 287 } 288 }); 289 } 290 291 private Response post( 292 final InputStream in, 293 final UserGroupInformation ugi, 294 final DelegationParam delegation, 295 final InetSocketAddress nnRpcAddr, 296 final String fullpath, 297 final PostOpParam op, 298 final BufferSizeParam bufferSize 299 ) throws IOException { 300 final DataNode datanode = (DataNode)context.getAttribute("datanode"); 301 302 switch(op.getValue()) { 303 case APPEND: 304 { 305 final Configuration conf = new Configuration(datanode.getConf()); 306 final int b = bufferSize.getValue(conf); 307 DFSClient dfsclient = new DFSClient(nnRpcAddr, conf); 308 FSDataOutputStream out = null; 309 try { 310 out = dfsclient.append(fullpath, b, null, null); 311 IOUtils.copyBytes(in, out, b); 312 out.close(); 313 out = null; 314 dfsclient.close(); 315 dfsclient = null; 316 } finally { 317 IOUtils.cleanup(LOG, out); 318 IOUtils.cleanup(LOG, dfsclient); 319 } 320 return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build(); 321 } 322 default: 323 throw new UnsupportedOperationException(op + " is not supported"); 324 } 325 } 326 327 /** Handle HTTP GET request for the root. */ 328 @GET 329 @Path("/") 330 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) 331 public Response getRoot( 332 @Context final UserGroupInformation ugi, 333 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) 334 final DelegationParam delegation, 335 @QueryParam(NamenodeRpcAddressParam.NAME) 336 @DefaultValue(NamenodeRpcAddressParam.DEFAULT) 337 final NamenodeRpcAddressParam namenodeRpcAddress, 338 @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT) 339 final GetOpParam op, 340 @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) 341 final OffsetParam offset, 342 @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT) 343 final LengthParam length, 344 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) 345 final BufferSizeParam bufferSize 346 ) throws IOException, InterruptedException { 347 return get(ugi, delegation, namenodeRpcAddress, ROOT, op, offset, length, 348 bufferSize); 349 } 350 351 /** Handle HTTP GET request. */ 352 @GET 353 @Path("{" + UriFsPathParam.NAME + ":.*}") 354 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) 355 public Response get( 356 @Context final UserGroupInformation ugi, 357 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) 358 final DelegationParam delegation, 359 @QueryParam(NamenodeRpcAddressParam.NAME) 360 @DefaultValue(NamenodeRpcAddressParam.DEFAULT) 361 final NamenodeRpcAddressParam namenodeRpcAddress, 362 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path, 363 @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT) 364 final GetOpParam op, 365 @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) 366 final OffsetParam offset, 367 @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT) 368 final LengthParam length, 369 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) 370 final BufferSizeParam bufferSize 371 ) throws IOException, InterruptedException { 372 373 final InetSocketAddress nnRpcAddr = namenodeRpcAddress.getValue(); 374 init(ugi, delegation, nnRpcAddr, path, op, offset, length, bufferSize); 375 376 return ugi.doAs(new PrivilegedExceptionAction<Response>() { 377 @Override 378 public Response run() throws IOException { 379 return get(ugi, delegation, nnRpcAddr, path.getAbsolutePath(), op, 380 offset, length, bufferSize); 381 } 382 }); 383 } 384 385 private Response get( 386 final UserGroupInformation ugi, 387 final DelegationParam delegation, 388 final InetSocketAddress nnRpcAddr, 389 final String fullpath, 390 final GetOpParam op, 391 final OffsetParam offset, 392 final LengthParam length, 393 final BufferSizeParam bufferSize 394 ) throws IOException { 395 final DataNode datanode = (DataNode)context.getAttribute("datanode"); 396 final Configuration conf = new Configuration(datanode.getConf()); 397 398 switch(op.getValue()) { 399 case OPEN: 400 { 401 final int b = bufferSize.getValue(conf); 402 final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf); 403 HdfsDataInputStream in = null; 404 try { 405 in = new HdfsDataInputStream(dfsclient.open(fullpath, b, true)); 406 in.seek(offset.getValue()); 407 } catch(IOException ioe) { 408 IOUtils.cleanup(LOG, in); 409 IOUtils.cleanup(LOG, dfsclient); 410 throw ioe; 411 } 412 413 final long n = length.getValue() != null ? 414 Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) : 415 in.getVisibleLength() - offset.getValue(); 416 417 /** 418 * Allow the Web UI to perform an AJAX request to get the data. 419 */ 420 return Response.ok(new OpenEntity(in, n, dfsclient)) 421 .type(MediaType.APPLICATION_OCTET_STREAM) 422 .header("Access-Control-Allow-Methods", "GET") 423 .header("Access-Control-Allow-Origin", "*") 424 .build(); 425 } 426 case GETFILECHECKSUM: 427 { 428 MD5MD5CRC32FileChecksum checksum = null; 429 DFSClient dfsclient = new DFSClient(nnRpcAddr, conf); 430 try { 431 checksum = dfsclient.getFileChecksum(fullpath); 432 dfsclient.close(); 433 dfsclient = null; 434 } finally { 435 IOUtils.cleanup(LOG, dfsclient); 436 } 437 final String js = JsonUtil.toJsonString(checksum); 438 return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); 439 } 440 default: 441 throw new UnsupportedOperationException(op + " is not supported"); 442 } 443 } 444}