001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.apache.hadoop.hdfs.server.namenode.web.resources; 019 020import java.io.FileNotFoundException; 021import java.io.IOException; 022import java.io.OutputStream; 023import java.io.OutputStreamWriter; 024import java.io.PrintWriter; 025import java.net.InetAddress; 026import java.net.URI; 027import java.net.URISyntaxException; 028import java.security.PrivilegedExceptionAction; 029import java.util.ArrayList; 030import java.util.EnumSet; 031 032import javax.servlet.ServletContext; 033import javax.servlet.http.HttpServletRequest; 034import javax.servlet.http.HttpServletResponse; 035import javax.ws.rs.Consumes; 036import javax.ws.rs.DELETE; 037import javax.ws.rs.DefaultValue; 038import javax.ws.rs.GET; 039import javax.ws.rs.POST; 040import javax.ws.rs.PUT; 041import javax.ws.rs.Path; 042import javax.ws.rs.PathParam; 043import javax.ws.rs.Produces; 044import javax.ws.rs.QueryParam; 045import javax.ws.rs.core.Context; 046import javax.ws.rs.core.MediaType; 047import javax.ws.rs.core.Response; 048import javax.ws.rs.core.StreamingOutput; 049 050import org.apache.commons.logging.Log; 051import org.apache.commons.logging.LogFactory; 052import org.apache.hadoop.conf.Configuration; 053import org.apache.hadoop.fs.ContentSummary; 054import org.apache.hadoop.fs.FileStatus; 055import org.apache.hadoop.fs.Options; 056import org.apache.hadoop.hdfs.StorageType; 057import org.apache.hadoop.hdfs.protocol.DatanodeInfo; 058import org.apache.hadoop.hdfs.protocol.DirectoryListing; 059import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; 060import org.apache.hadoop.hdfs.protocol.LocatedBlocks; 061import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; 062import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; 063import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; 064import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; 065import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo; 066import org.apache.hadoop.hdfs.server.common.JspHelper; 067import org.apache.hadoop.hdfs.server.namenode.NameNode; 068import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; 069import org.apache.hadoop.hdfs.web.JsonUtil; 070import org.apache.hadoop.hdfs.web.ParamFilter; 071import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem; 072import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; 073import org.apache.hadoop.hdfs.web.resources.AccessTimeParam; 074import org.apache.hadoop.hdfs.web.resources.BlockSizeParam; 075import org.apache.hadoop.hdfs.web.resources.BufferSizeParam; 076import org.apache.hadoop.hdfs.web.resources.ConcatSourcesParam; 077import org.apache.hadoop.hdfs.web.resources.CreateParentParam; 078import org.apache.hadoop.hdfs.web.resources.DelegationParam; 079import org.apache.hadoop.hdfs.web.resources.DeleteOpParam; 080import org.apache.hadoop.hdfs.web.resources.DestinationParam; 081import org.apache.hadoop.hdfs.web.resources.DoAsParam; 082import org.apache.hadoop.hdfs.web.resources.GetOpParam; 083import org.apache.hadoop.hdfs.web.resources.GroupParam; 084import org.apache.hadoop.hdfs.web.resources.HttpOpParam; 085import org.apache.hadoop.hdfs.web.resources.LengthParam; 086import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam; 087import org.apache.hadoop.hdfs.web.resources.NamenodeRpcAddressParam; 088import org.apache.hadoop.hdfs.web.resources.OffsetParam; 089import org.apache.hadoop.hdfs.web.resources.OverwriteParam; 090import org.apache.hadoop.hdfs.web.resources.OwnerParam; 091import org.apache.hadoop.hdfs.web.resources.Param; 092import org.apache.hadoop.hdfs.web.resources.PermissionParam; 093import org.apache.hadoop.hdfs.web.resources.PostOpParam; 094import org.apache.hadoop.hdfs.web.resources.PutOpParam; 095import org.apache.hadoop.hdfs.web.resources.RecursiveParam; 096import org.apache.hadoop.hdfs.web.resources.RenameOptionSetParam; 097import org.apache.hadoop.hdfs.web.resources.RenewerParam; 098import org.apache.hadoop.hdfs.web.resources.ReplicationParam; 099import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam; 100import org.apache.hadoop.hdfs.web.resources.UriFsPathParam; 101import org.apache.hadoop.hdfs.web.resources.UserParam; 102import org.apache.hadoop.io.Text; 103import org.apache.hadoop.ipc.Server; 104import org.apache.hadoop.net.NodeBase; 105import org.apache.hadoop.security.Credentials; 106import org.apache.hadoop.security.UserGroupInformation; 107import org.apache.hadoop.security.token.Token; 108import org.apache.hadoop.security.token.TokenIdentifier; 109 110import com.google.common.base.Charsets; 111import com.sun.jersey.spi.container.ResourceFilters; 112 113/** Web-hdfs NameNode implementation. */ 114@Path("") 115@ResourceFilters(ParamFilter.class) 116public class NamenodeWebHdfsMethods { 117 public static final Log LOG = LogFactory.getLog(NamenodeWebHdfsMethods.class); 118 119 private static final UriFsPathParam ROOT = new UriFsPathParam(""); 120 121 private static final ThreadLocal<String> REMOTE_ADDRESS = new ThreadLocal<String>(); 122 123 /** @return the remote client address. */ 124 public static String getRemoteAddress() { 125 return REMOTE_ADDRESS.get(); 126 } 127 128 public static InetAddress getRemoteIp() { 129 try { 130 return InetAddress.getByName(getRemoteAddress()); 131 } catch (Exception e) { 132 return null; 133 } 134 } 135 136 /** 137 * Returns true if a WebHdfs request is in progress. Akin to 138 * {@link Server#isRpcInvocation()}. 139 */ 140 public static boolean isWebHdfsInvocation() { 141 return getRemoteAddress() != null; 142 } 143 144 private @Context ServletContext context; 145 private @Context HttpServletRequest request; 146 private @Context HttpServletResponse response; 147 148 private void init(final UserGroupInformation ugi, 149 final DelegationParam delegation, 150 final UserParam username, final DoAsParam doAsUser, 151 final UriFsPathParam path, final HttpOpParam<?> op, 152 final Param<?, ?>... parameters) { 153 if (LOG.isTraceEnabled()) { 154 LOG.trace("HTTP " + op.getValue().getType() + ": " + op + ", " + path 155 + ", ugi=" + ugi + ", " + username + ", " + doAsUser 156 + Param.toSortedString(", ", parameters)); 157 } 158 159 //clear content type 160 response.setContentType(null); 161 } 162 163 static DatanodeInfo chooseDatanode(final NameNode namenode, 164 final String path, final HttpOpParam.Op op, final long openOffset, 165 final long blocksize, final Configuration conf) throws IOException { 166 final BlockManager bm = namenode.getNamesystem().getBlockManager(); 167 168 if (op == PutOpParam.Op.CREATE) { 169 //choose a datanode near to client 170 final DatanodeDescriptor clientNode = bm.getDatanodeManager( 171 ).getDatanodeByHost(getRemoteAddress()); 172 if (clientNode != null) { 173 final DatanodeStorageInfo[] storages = bm.getBlockPlacementPolicy() 174 .chooseTarget(path, 1, clientNode, 175 new ArrayList<DatanodeStorageInfo>(), false, null, blocksize, 176 // TODO: get storage type from the file 177 StorageType.DEFAULT); 178 if (storages.length > 0) { 179 return storages[0].getDatanodeDescriptor(); 180 } 181 } 182 } else if (op == GetOpParam.Op.OPEN 183 || op == GetOpParam.Op.GETFILECHECKSUM 184 || op == PostOpParam.Op.APPEND) { 185 //choose a datanode containing a replica 186 final NamenodeProtocols np = namenode.getRpcServer(); 187 final HdfsFileStatus status = np.getFileInfo(path); 188 if (status == null) { 189 throw new FileNotFoundException("File " + path + " not found."); 190 } 191 final long len = status.getLen(); 192 if (op == GetOpParam.Op.OPEN) { 193 if (openOffset < 0L || (openOffset >= len && len > 0)) { 194 throw new IOException("Offset=" + openOffset 195 + " out of the range [0, " + len + "); " + op + ", path=" + path); 196 } 197 } 198 199 if (len > 0) { 200 final long offset = op == GetOpParam.Op.OPEN? openOffset: len - 1; 201 final LocatedBlocks locations = np.getBlockLocations(path, offset, 1); 202 final int count = locations.locatedBlockCount(); 203 if (count > 0) { 204 return JspHelper.bestNode(locations.get(0).getLocations(), false, conf); 205 } 206 } 207 } 208 209 return (DatanodeDescriptor)bm.getDatanodeManager().getNetworkTopology( 210 ).chooseRandom(NodeBase.ROOT); 211 } 212 213 private Token<? extends TokenIdentifier> generateDelegationToken( 214 final NameNode namenode, final UserGroupInformation ugi, 215 final String renewer) throws IOException { 216 final Credentials c = DelegationTokenSecretManager.createCredentials( 217 namenode, ugi, renewer != null? renewer: ugi.getShortUserName()); 218 final Token<? extends TokenIdentifier> t = c.getAllTokens().iterator().next(); 219 Text kind = request.getScheme().equals("http") ? WebHdfsFileSystem.TOKEN_KIND : SWebHdfsFileSystem.TOKEN_KIND; 220 t.setKind(kind); 221 return t; 222 } 223 224 private URI redirectURI(final NameNode namenode, 225 final UserGroupInformation ugi, final DelegationParam delegation, 226 final UserParam username, final DoAsParam doAsUser, 227 final String path, final HttpOpParam.Op op, final long openOffset, 228 final long blocksize, 229 final Param<?, ?>... parameters) throws URISyntaxException, IOException { 230 final Configuration conf = (Configuration)context.getAttribute(JspHelper.CURRENT_CONF); 231 final DatanodeInfo dn = chooseDatanode(namenode, path, op, openOffset, 232 blocksize, conf); 233 234 final String delegationQuery; 235 if (!UserGroupInformation.isSecurityEnabled()) { 236 //security disabled 237 delegationQuery = Param.toSortedString("&", doAsUser, username); 238 } else if (delegation.getValue() != null) { 239 //client has provided a token 240 delegationQuery = "&" + delegation; 241 } else { 242 //generate a token 243 final Token<? extends TokenIdentifier> t = generateDelegationToken( 244 namenode, ugi, request.getUserPrincipal().getName()); 245 delegationQuery = "&" + new DelegationParam(t.encodeToUrlString()); 246 } 247 final String query = op.toQueryString() + delegationQuery 248 + "&" + new NamenodeRpcAddressParam(namenode) 249 + Param.toSortedString("&", parameters); 250 final String uripath = WebHdfsFileSystem.PATH_PREFIX + path; 251 252 final String scheme = request.getScheme(); 253 int port = "http".equals(scheme) ? dn.getInfoPort() : dn 254 .getInfoSecurePort(); 255 final URI uri = new URI(scheme, null, dn.getHostName(), port, uripath, 256 query, null); 257 258 if (LOG.isTraceEnabled()) { 259 LOG.trace("redirectURI=" + uri); 260 } 261 return uri; 262 } 263 264 /** Handle HTTP PUT request for the root. */ 265 @PUT 266 @Path("/") 267 @Consumes({"*/*"}) 268 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) 269 public Response putRoot( 270 @Context final UserGroupInformation ugi, 271 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) 272 final DelegationParam delegation, 273 @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) 274 final UserParam username, 275 @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) 276 final DoAsParam doAsUser, 277 @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT) 278 final PutOpParam op, 279 @QueryParam(DestinationParam.NAME) @DefaultValue(DestinationParam.DEFAULT) 280 final DestinationParam destination, 281 @QueryParam(OwnerParam.NAME) @DefaultValue(OwnerParam.DEFAULT) 282 final OwnerParam owner, 283 @QueryParam(GroupParam.NAME) @DefaultValue(GroupParam.DEFAULT) 284 final GroupParam group, 285 @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT) 286 final PermissionParam permission, 287 @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT) 288 final OverwriteParam overwrite, 289 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) 290 final BufferSizeParam bufferSize, 291 @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT) 292 final ReplicationParam replication, 293 @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT) 294 final BlockSizeParam blockSize, 295 @QueryParam(ModificationTimeParam.NAME) @DefaultValue(ModificationTimeParam.DEFAULT) 296 final ModificationTimeParam modificationTime, 297 @QueryParam(AccessTimeParam.NAME) @DefaultValue(AccessTimeParam.DEFAULT) 298 final AccessTimeParam accessTime, 299 @QueryParam(RenameOptionSetParam.NAME) @DefaultValue(RenameOptionSetParam.DEFAULT) 300 final RenameOptionSetParam renameOptions, 301 @QueryParam(CreateParentParam.NAME) @DefaultValue(CreateParentParam.DEFAULT) 302 final CreateParentParam createParent, 303 @QueryParam(TokenArgumentParam.NAME) @DefaultValue(TokenArgumentParam.DEFAULT) 304 final TokenArgumentParam delegationTokenArgument 305 ) throws IOException, InterruptedException { 306 return put(ugi, delegation, username, doAsUser, ROOT, op, destination, 307 owner, group, permission, overwrite, bufferSize, replication, 308 blockSize, modificationTime, accessTime, renameOptions, createParent, 309 delegationTokenArgument); 310 } 311 312 /** Handle HTTP PUT request. */ 313 @PUT 314 @Path("{" + UriFsPathParam.NAME + ":.*}") 315 @Consumes({"*/*"}) 316 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) 317 public Response put( 318 @Context final UserGroupInformation ugi, 319 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) 320 final DelegationParam delegation, 321 @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) 322 final UserParam username, 323 @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) 324 final DoAsParam doAsUser, 325 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path, 326 @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT) 327 final PutOpParam op, 328 @QueryParam(DestinationParam.NAME) @DefaultValue(DestinationParam.DEFAULT) 329 final DestinationParam destination, 330 @QueryParam(OwnerParam.NAME) @DefaultValue(OwnerParam.DEFAULT) 331 final OwnerParam owner, 332 @QueryParam(GroupParam.NAME) @DefaultValue(GroupParam.DEFAULT) 333 final GroupParam group, 334 @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT) 335 final PermissionParam permission, 336 @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT) 337 final OverwriteParam overwrite, 338 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) 339 final BufferSizeParam bufferSize, 340 @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT) 341 final ReplicationParam replication, 342 @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT) 343 final BlockSizeParam blockSize, 344 @QueryParam(ModificationTimeParam.NAME) @DefaultValue(ModificationTimeParam.DEFAULT) 345 final ModificationTimeParam modificationTime, 346 @QueryParam(AccessTimeParam.NAME) @DefaultValue(AccessTimeParam.DEFAULT) 347 final AccessTimeParam accessTime, 348 @QueryParam(RenameOptionSetParam.NAME) @DefaultValue(RenameOptionSetParam.DEFAULT) 349 final RenameOptionSetParam renameOptions, 350 @QueryParam(CreateParentParam.NAME) @DefaultValue(CreateParentParam.DEFAULT) 351 final CreateParentParam createParent, 352 @QueryParam(TokenArgumentParam.NAME) @DefaultValue(TokenArgumentParam.DEFAULT) 353 final TokenArgumentParam delegationTokenArgument 354 ) throws IOException, InterruptedException { 355 356 init(ugi, delegation, username, doAsUser, path, op, destination, owner, 357 group, permission, overwrite, bufferSize, replication, blockSize, 358 modificationTime, accessTime, renameOptions, delegationTokenArgument); 359 360 return ugi.doAs(new PrivilegedExceptionAction<Response>() { 361 @Override 362 public Response run() throws IOException, URISyntaxException { 363 REMOTE_ADDRESS.set(request.getRemoteAddr()); 364 try { 365 return put(ugi, delegation, username, doAsUser, 366 path.getAbsolutePath(), op, destination, owner, group, 367 permission, overwrite, bufferSize, replication, blockSize, 368 modificationTime, accessTime, renameOptions, createParent, 369 delegationTokenArgument); 370 } finally { 371 REMOTE_ADDRESS.set(null); 372 } 373 } 374 }); 375 } 376 377 private Response put( 378 final UserGroupInformation ugi, 379 final DelegationParam delegation, 380 final UserParam username, 381 final DoAsParam doAsUser, 382 final String fullpath, 383 final PutOpParam op, 384 final DestinationParam destination, 385 final OwnerParam owner, 386 final GroupParam group, 387 final PermissionParam permission, 388 final OverwriteParam overwrite, 389 final BufferSizeParam bufferSize, 390 final ReplicationParam replication, 391 final BlockSizeParam blockSize, 392 final ModificationTimeParam modificationTime, 393 final AccessTimeParam accessTime, 394 final RenameOptionSetParam renameOptions, 395 final CreateParentParam createParent, 396 final TokenArgumentParam delegationTokenArgument 397 ) throws IOException, URISyntaxException { 398 399 final Configuration conf = (Configuration)context.getAttribute(JspHelper.CURRENT_CONF); 400 final NameNode namenode = (NameNode)context.getAttribute("name.node"); 401 final NamenodeProtocols np = namenode.getRpcServer(); 402 403 switch(op.getValue()) { 404 case CREATE: 405 { 406 final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser, 407 fullpath, op.getValue(), -1L, blockSize.getValue(conf), 408 permission, overwrite, bufferSize, replication, blockSize); 409 return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build(); 410 } 411 case MKDIRS: 412 { 413 final boolean b = np.mkdirs(fullpath, permission.getFsPermission(), true); 414 final String js = JsonUtil.toJsonString("boolean", b); 415 return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); 416 } 417 case CREATESYMLINK: 418 { 419 np.createSymlink(destination.getValue(), fullpath, 420 PermissionParam.getDefaultFsPermission(), createParent.getValue()); 421 return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build(); 422 } 423 case RENAME: 424 { 425 final EnumSet<Options.Rename> s = renameOptions.getValue(); 426 if (s.isEmpty()) { 427 final boolean b = np.rename(fullpath, destination.getValue()); 428 final String js = JsonUtil.toJsonString("boolean", b); 429 return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); 430 } else { 431 np.rename2(fullpath, destination.getValue(), 432 s.toArray(new Options.Rename[s.size()])); 433 return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build(); 434 } 435 } 436 case SETREPLICATION: 437 { 438 final boolean b = np.setReplication(fullpath, replication.getValue(conf)); 439 final String js = JsonUtil.toJsonString("boolean", b); 440 return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); 441 } 442 case SETOWNER: 443 { 444 if (owner.getValue() == null && group.getValue() == null) { 445 throw new IllegalArgumentException("Both owner and group are empty."); 446 } 447 448 np.setOwner(fullpath, owner.getValue(), group.getValue()); 449 return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build(); 450 } 451 case SETPERMISSION: 452 { 453 np.setPermission(fullpath, permission.getFsPermission()); 454 return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build(); 455 } 456 case SETTIMES: 457 { 458 np.setTimes(fullpath, modificationTime.getValue(), accessTime.getValue()); 459 return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build(); 460 } 461 case RENEWDELEGATIONTOKEN: 462 { 463 final Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(); 464 token.decodeFromUrlString(delegationTokenArgument.getValue()); 465 final long expiryTime = np.renewDelegationToken(token); 466 final String js = JsonUtil.toJsonString("long", expiryTime); 467 return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); 468 } 469 case CANCELDELEGATIONTOKEN: 470 { 471 final Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(); 472 token.decodeFromUrlString(delegationTokenArgument.getValue()); 473 np.cancelDelegationToken(token); 474 return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build(); 475 } 476 default: 477 throw new UnsupportedOperationException(op + " is not supported"); 478 } 479 } 480 481 /** Handle HTTP POST request for the root. */ 482 @POST 483 @Path("/") 484 @Consumes({"*/*"}) 485 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) 486 public Response postRoot( 487 @Context final UserGroupInformation ugi, 488 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) 489 final DelegationParam delegation, 490 @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) 491 final UserParam username, 492 @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) 493 final DoAsParam doAsUser, 494 @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT) 495 final PostOpParam op, 496 @QueryParam(ConcatSourcesParam.NAME) @DefaultValue(ConcatSourcesParam.DEFAULT) 497 final ConcatSourcesParam concatSrcs, 498 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) 499 final BufferSizeParam bufferSize 500 ) throws IOException, InterruptedException { 501 return post(ugi, delegation, username, doAsUser, ROOT, op, concatSrcs, bufferSize); 502 } 503 504 /** Handle HTTP POST request. */ 505 @POST 506 @Path("{" + UriFsPathParam.NAME + ":.*}") 507 @Consumes({"*/*"}) 508 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) 509 public Response post( 510 @Context final UserGroupInformation ugi, 511 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) 512 final DelegationParam delegation, 513 @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) 514 final UserParam username, 515 @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) 516 final DoAsParam doAsUser, 517 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path, 518 @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT) 519 final PostOpParam op, 520 @QueryParam(ConcatSourcesParam.NAME) @DefaultValue(ConcatSourcesParam.DEFAULT) 521 final ConcatSourcesParam concatSrcs, 522 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) 523 final BufferSizeParam bufferSize 524 ) throws IOException, InterruptedException { 525 526 init(ugi, delegation, username, doAsUser, path, op, concatSrcs, bufferSize); 527 528 return ugi.doAs(new PrivilegedExceptionAction<Response>() { 529 @Override 530 public Response run() throws IOException, URISyntaxException { 531 REMOTE_ADDRESS.set(request.getRemoteAddr()); 532 try { 533 return post(ugi, delegation, username, doAsUser, 534 path.getAbsolutePath(), op, concatSrcs, bufferSize); 535 } finally { 536 REMOTE_ADDRESS.set(null); 537 } 538 } 539 }); 540 } 541 542 private Response post( 543 final UserGroupInformation ugi, 544 final DelegationParam delegation, 545 final UserParam username, 546 final DoAsParam doAsUser, 547 final String fullpath, 548 final PostOpParam op, 549 final ConcatSourcesParam concatSrcs, 550 final BufferSizeParam bufferSize 551 ) throws IOException, URISyntaxException { 552 final NameNode namenode = (NameNode)context.getAttribute("name.node"); 553 554 switch(op.getValue()) { 555 case APPEND: 556 { 557 final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser, 558 fullpath, op.getValue(), -1L, -1L, bufferSize); 559 return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build(); 560 } 561 case CONCAT: 562 { 563 namenode.getRpcServer().concat(fullpath, concatSrcs.getAbsolutePaths()); 564 return Response.ok().build(); 565 } 566 default: 567 throw new UnsupportedOperationException(op + " is not supported"); 568 } 569 } 570 571 /** Handle HTTP GET request for the root. */ 572 @GET 573 @Path("/") 574 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) 575 public Response getRoot( 576 @Context final UserGroupInformation ugi, 577 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) 578 final DelegationParam delegation, 579 @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) 580 final UserParam username, 581 @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) 582 final DoAsParam doAsUser, 583 @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT) 584 final GetOpParam op, 585 @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) 586 final OffsetParam offset, 587 @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT) 588 final LengthParam length, 589 @QueryParam(RenewerParam.NAME) @DefaultValue(RenewerParam.DEFAULT) 590 final RenewerParam renewer, 591 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) 592 final BufferSizeParam bufferSize 593 ) throws IOException, InterruptedException { 594 return get(ugi, delegation, username, doAsUser, ROOT, op, 595 offset, length, renewer, bufferSize); 596 } 597 598 /** Handle HTTP GET request. */ 599 @GET 600 @Path("{" + UriFsPathParam.NAME + ":.*}") 601 @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON}) 602 public Response get( 603 @Context final UserGroupInformation ugi, 604 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) 605 final DelegationParam delegation, 606 @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) 607 final UserParam username, 608 @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) 609 final DoAsParam doAsUser, 610 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path, 611 @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT) 612 final GetOpParam op, 613 @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT) 614 final OffsetParam offset, 615 @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT) 616 final LengthParam length, 617 @QueryParam(RenewerParam.NAME) @DefaultValue(RenewerParam.DEFAULT) 618 final RenewerParam renewer, 619 @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT) 620 final BufferSizeParam bufferSize 621 ) throws IOException, InterruptedException { 622 623 init(ugi, delegation, username, doAsUser, path, op, 624 offset, length, renewer, bufferSize); 625 626 return ugi.doAs(new PrivilegedExceptionAction<Response>() { 627 @Override 628 public Response run() throws IOException, URISyntaxException { 629 REMOTE_ADDRESS.set(request.getRemoteAddr()); 630 try { 631 return get(ugi, delegation, username, doAsUser, 632 path.getAbsolutePath(), op, offset, length, renewer, bufferSize); 633 } finally { 634 REMOTE_ADDRESS.set(null); 635 } 636 } 637 }); 638 } 639 640 private Response get( 641 final UserGroupInformation ugi, 642 final DelegationParam delegation, 643 final UserParam username, 644 final DoAsParam doAsUser, 645 final String fullpath, 646 final GetOpParam op, 647 final OffsetParam offset, 648 final LengthParam length, 649 final RenewerParam renewer, 650 final BufferSizeParam bufferSize 651 ) throws IOException, URISyntaxException { 652 final NameNode namenode = (NameNode)context.getAttribute("name.node"); 653 final NamenodeProtocols np = namenode.getRpcServer(); 654 655 switch(op.getValue()) { 656 case OPEN: 657 { 658 final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser, 659 fullpath, op.getValue(), offset.getValue(), -1L, offset, length, bufferSize); 660 return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build(); 661 } 662 case GET_BLOCK_LOCATIONS: 663 { 664 final long offsetValue = offset.getValue(); 665 final Long lengthValue = length.getValue(); 666 final LocatedBlocks locatedblocks = np.getBlockLocations(fullpath, 667 offsetValue, lengthValue != null? lengthValue: Long.MAX_VALUE); 668 final String js = JsonUtil.toJsonString(locatedblocks); 669 return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); 670 } 671 case GETFILESTATUS: 672 { 673 final HdfsFileStatus status = np.getFileInfo(fullpath); 674 if (status == null) { 675 throw new FileNotFoundException("File does not exist: " + fullpath); 676 } 677 678 final String js = JsonUtil.toJsonString(status, true); 679 return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); 680 } 681 case LISTSTATUS: 682 { 683 final StreamingOutput streaming = getListingStream(np, fullpath); 684 return Response.ok(streaming).type(MediaType.APPLICATION_JSON).build(); 685 } 686 case GETCONTENTSUMMARY: 687 { 688 final ContentSummary contentsummary = np.getContentSummary(fullpath); 689 final String js = JsonUtil.toJsonString(contentsummary); 690 return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); 691 } 692 case GETFILECHECKSUM: 693 { 694 final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser, 695 fullpath, op.getValue(), -1L, -1L); 696 return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build(); 697 } 698 case GETDELEGATIONTOKEN: 699 { 700 if (delegation.getValue() != null) { 701 throw new IllegalArgumentException(delegation.getName() 702 + " parameter is not null."); 703 } 704 final Token<? extends TokenIdentifier> token = generateDelegationToken( 705 namenode, ugi, renewer.getValue()); 706 final String js = JsonUtil.toJsonString(token); 707 return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); 708 } 709 case GETHOMEDIRECTORY: 710 { 711 final String js = JsonUtil.toJsonString( 712 org.apache.hadoop.fs.Path.class.getSimpleName(), 713 WebHdfsFileSystem.getHomeDirectoryString(ugi)); 714 return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); 715 } 716 default: 717 throw new UnsupportedOperationException(op + " is not supported"); 718 } 719 } 720 721 private static DirectoryListing getDirectoryListing(final NamenodeProtocols np, 722 final String p, byte[] startAfter) throws IOException { 723 final DirectoryListing listing = np.getListing(p, startAfter, false); 724 if (listing == null) { // the directory does not exist 725 throw new FileNotFoundException("File " + p + " does not exist."); 726 } 727 return listing; 728 } 729 730 private static StreamingOutput getListingStream(final NamenodeProtocols np, 731 final String p) throws IOException { 732 // allows exceptions like FNF or ACE to prevent http response of 200 for 733 // a failure since we can't (currently) return error responses in the 734 // middle of a streaming operation 735 final DirectoryListing firstDirList = getDirectoryListing(np, p, 736 HdfsFileStatus.EMPTY_NAME); 737 738 // must save ugi because the streaming object will be executed outside 739 // the remote user's ugi 740 final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); 741 return new StreamingOutput() { 742 @Override 743 public void write(final OutputStream outstream) throws IOException { 744 final PrintWriter out = new PrintWriter(new OutputStreamWriter( 745 outstream, Charsets.UTF_8)); 746 out.println("{\"" + FileStatus.class.getSimpleName() + "es\":{\"" 747 + FileStatus.class.getSimpleName() + "\":["); 748 749 try { 750 // restore remote user's ugi 751 ugi.doAs(new PrivilegedExceptionAction<Void>() { 752 @Override 753 public Void run() throws IOException { 754 long n = 0; 755 for (DirectoryListing dirList = firstDirList; ; 756 dirList = getDirectoryListing(np, p, dirList.getLastName()) 757 ) { 758 // send each segment of the directory listing 759 for (HdfsFileStatus s : dirList.getPartialListing()) { 760 if (n++ > 0) { 761 out.println(','); 762 } 763 out.print(JsonUtil.toJsonString(s, false)); 764 } 765 // stop if last segment 766 if (!dirList.hasMore()) { 767 break; 768 } 769 } 770 return null; 771 } 772 }); 773 } catch (InterruptedException e) { 774 throw new IOException(e); 775 } 776 777 out.println(); 778 out.println("]}}"); 779 out.flush(); 780 } 781 }; 782 } 783 784 /** Handle HTTP DELETE request for the root. */ 785 @DELETE 786 @Path("/") 787 @Produces(MediaType.APPLICATION_JSON) 788 public Response deleteRoot( 789 @Context final UserGroupInformation ugi, 790 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) 791 final DelegationParam delegation, 792 @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) 793 final UserParam username, 794 @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) 795 final DoAsParam doAsUser, 796 @QueryParam(DeleteOpParam.NAME) @DefaultValue(DeleteOpParam.DEFAULT) 797 final DeleteOpParam op, 798 @QueryParam(RecursiveParam.NAME) @DefaultValue(RecursiveParam.DEFAULT) 799 final RecursiveParam recursive 800 ) throws IOException, InterruptedException { 801 return delete(ugi, delegation, username, doAsUser, ROOT, op, recursive); 802 } 803 804 /** Handle HTTP DELETE request. */ 805 @DELETE 806 @Path("{" + UriFsPathParam.NAME + ":.*}") 807 @Produces(MediaType.APPLICATION_JSON) 808 public Response delete( 809 @Context final UserGroupInformation ugi, 810 @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT) 811 final DelegationParam delegation, 812 @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT) 813 final UserParam username, 814 @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT) 815 final DoAsParam doAsUser, 816 @PathParam(UriFsPathParam.NAME) final UriFsPathParam path, 817 @QueryParam(DeleteOpParam.NAME) @DefaultValue(DeleteOpParam.DEFAULT) 818 final DeleteOpParam op, 819 @QueryParam(RecursiveParam.NAME) @DefaultValue(RecursiveParam.DEFAULT) 820 final RecursiveParam recursive 821 ) throws IOException, InterruptedException { 822 823 init(ugi, delegation, username, doAsUser, path, op, recursive); 824 825 return ugi.doAs(new PrivilegedExceptionAction<Response>() { 826 @Override 827 public Response run() throws IOException { 828 REMOTE_ADDRESS.set(request.getRemoteAddr()); 829 try { 830 return delete(ugi, delegation, username, doAsUser, 831 path.getAbsolutePath(), op, recursive); 832 } finally { 833 REMOTE_ADDRESS.set(null); 834 } 835 } 836 }); 837 } 838 839 private Response delete( 840 final UserGroupInformation ugi, 841 final DelegationParam delegation, 842 final UserParam username, 843 final DoAsParam doAsUser, 844 final String fullpath, 845 final DeleteOpParam op, 846 final RecursiveParam recursive 847 ) throws IOException { 848 final NameNode namenode = (NameNode)context.getAttribute("name.node"); 849 850 switch(op.getValue()) { 851 case DELETE: 852 { 853 final boolean b = namenode.getRpcServer().delete(fullpath, recursive.getValue()); 854 final String js = JsonUtil.toJsonString("boolean", b); 855 return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); 856 } 857 default: 858 throw new UnsupportedOperationException(op + " is not supported"); 859 } 860 } 861}