001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs.server.namenode.web.resources;
019
020import java.io.FileNotFoundException;
021import java.io.IOException;
022import java.io.OutputStream;
023import java.io.OutputStreamWriter;
024import java.io.PrintWriter;
025import java.net.InetAddress;
026import java.net.URI;
027import java.net.URISyntaxException;
028import java.security.PrivilegedExceptionAction;
029import java.util.ArrayList;
030import java.util.EnumSet;
031
032import javax.servlet.ServletContext;
033import javax.servlet.http.HttpServletRequest;
034import javax.servlet.http.HttpServletResponse;
035import javax.ws.rs.Consumes;
036import javax.ws.rs.DELETE;
037import javax.ws.rs.DefaultValue;
038import javax.ws.rs.GET;
039import javax.ws.rs.POST;
040import javax.ws.rs.PUT;
041import javax.ws.rs.Path;
042import javax.ws.rs.PathParam;
043import javax.ws.rs.Produces;
044import javax.ws.rs.QueryParam;
045import javax.ws.rs.core.Context;
046import javax.ws.rs.core.MediaType;
047import javax.ws.rs.core.Response;
048import javax.ws.rs.core.StreamingOutput;
049
050import org.apache.commons.logging.Log;
051import org.apache.commons.logging.LogFactory;
052import org.apache.hadoop.conf.Configuration;
053import org.apache.hadoop.fs.ContentSummary;
054import org.apache.hadoop.fs.FileStatus;
055import org.apache.hadoop.fs.Options;
056import org.apache.hadoop.hdfs.StorageType;
057import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
058import org.apache.hadoop.hdfs.protocol.DirectoryListing;
059import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
060import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
061import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
062import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
063import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
064import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
065import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
066import org.apache.hadoop.hdfs.server.common.JspHelper;
067import org.apache.hadoop.hdfs.server.namenode.NameNode;
068import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
069import org.apache.hadoop.hdfs.web.JsonUtil;
070import org.apache.hadoop.hdfs.web.ParamFilter;
071import org.apache.hadoop.hdfs.web.SWebHdfsFileSystem;
072import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
073import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
074import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
075import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
076import org.apache.hadoop.hdfs.web.resources.ConcatSourcesParam;
077import org.apache.hadoop.hdfs.web.resources.CreateParentParam;
078import org.apache.hadoop.hdfs.web.resources.DelegationParam;
079import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
080import org.apache.hadoop.hdfs.web.resources.DestinationParam;
081import org.apache.hadoop.hdfs.web.resources.DoAsParam;
082import org.apache.hadoop.hdfs.web.resources.GetOpParam;
083import org.apache.hadoop.hdfs.web.resources.GroupParam;
084import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
085import org.apache.hadoop.hdfs.web.resources.LengthParam;
086import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam;
087import org.apache.hadoop.hdfs.web.resources.NamenodeRpcAddressParam;
088import org.apache.hadoop.hdfs.web.resources.OffsetParam;
089import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
090import org.apache.hadoop.hdfs.web.resources.OwnerParam;
091import org.apache.hadoop.hdfs.web.resources.Param;
092import org.apache.hadoop.hdfs.web.resources.PermissionParam;
093import org.apache.hadoop.hdfs.web.resources.PostOpParam;
094import org.apache.hadoop.hdfs.web.resources.PutOpParam;
095import org.apache.hadoop.hdfs.web.resources.RecursiveParam;
096import org.apache.hadoop.hdfs.web.resources.RenameOptionSetParam;
097import org.apache.hadoop.hdfs.web.resources.RenewerParam;
098import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
099import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
100import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
101import org.apache.hadoop.hdfs.web.resources.UserParam;
102import org.apache.hadoop.io.Text;
103import org.apache.hadoop.ipc.Server;
104import org.apache.hadoop.net.NodeBase;
105import org.apache.hadoop.security.Credentials;
106import org.apache.hadoop.security.UserGroupInformation;
107import org.apache.hadoop.security.token.Token;
108import org.apache.hadoop.security.token.TokenIdentifier;
109
110import com.google.common.base.Charsets;
111import com.sun.jersey.spi.container.ResourceFilters;
112
113/** Web-hdfs NameNode implementation. */
114@Path("")
115@ResourceFilters(ParamFilter.class)
116public class NamenodeWebHdfsMethods {
117  public static final Log LOG = LogFactory.getLog(NamenodeWebHdfsMethods.class);
118
119  private static final UriFsPathParam ROOT = new UriFsPathParam("");
120  
121  private static final ThreadLocal<String> REMOTE_ADDRESS = new ThreadLocal<String>(); 
122
123  /** @return the remote client address. */
124  public static String getRemoteAddress() {
125    return REMOTE_ADDRESS.get();
126  }
127
128  public static InetAddress getRemoteIp() {
129    try {
130      return InetAddress.getByName(getRemoteAddress());
131    } catch (Exception e) {
132      return null;
133    }
134  }
135
136  /**
137   * Returns true if a WebHdfs request is in progress.  Akin to
138   * {@link Server#isRpcInvocation()}.
139   */
140  public static boolean isWebHdfsInvocation() {
141    return getRemoteAddress() != null;
142  }
143
144  private @Context ServletContext context;
145  private @Context HttpServletRequest request;
146  private @Context HttpServletResponse response;
147
148  private void init(final UserGroupInformation ugi,
149      final DelegationParam delegation,
150      final UserParam username, final DoAsParam doAsUser,
151      final UriFsPathParam path, final HttpOpParam<?> op,
152      final Param<?, ?>... parameters) {
153    if (LOG.isTraceEnabled()) {
154      LOG.trace("HTTP " + op.getValue().getType() + ": " + op + ", " + path
155          + ", ugi=" + ugi + ", " + username + ", " + doAsUser
156          + Param.toSortedString(", ", parameters));
157    }
158
159    //clear content type
160    response.setContentType(null);
161  }
162
163  static DatanodeInfo chooseDatanode(final NameNode namenode,
164      final String path, final HttpOpParam.Op op, final long openOffset,
165      final long blocksize, final Configuration conf) throws IOException {
166    final BlockManager bm = namenode.getNamesystem().getBlockManager();
167
168    if (op == PutOpParam.Op.CREATE) {
169      //choose a datanode near to client 
170      final DatanodeDescriptor clientNode = bm.getDatanodeManager(
171          ).getDatanodeByHost(getRemoteAddress());
172      if (clientNode != null) {
173        final DatanodeStorageInfo[] storages = bm.getBlockPlacementPolicy()
174            .chooseTarget(path, 1, clientNode,
175                new ArrayList<DatanodeStorageInfo>(), false, null, blocksize,
176                // TODO: get storage type from the file
177                StorageType.DEFAULT);
178        if (storages.length > 0) {
179          return storages[0].getDatanodeDescriptor();
180        }
181      }
182    } else if (op == GetOpParam.Op.OPEN
183        || op == GetOpParam.Op.GETFILECHECKSUM
184        || op == PostOpParam.Op.APPEND) {
185      //choose a datanode containing a replica 
186      final NamenodeProtocols np = namenode.getRpcServer();
187      final HdfsFileStatus status = np.getFileInfo(path);
188      if (status == null) {
189        throw new FileNotFoundException("File " + path + " not found.");
190      }
191      final long len = status.getLen();
192      if (op == GetOpParam.Op.OPEN) {
193        if (openOffset < 0L || (openOffset >= len && len > 0)) {
194          throw new IOException("Offset=" + openOffset
195              + " out of the range [0, " + len + "); " + op + ", path=" + path);
196        }
197      }
198
199      if (len > 0) {
200        final long offset = op == GetOpParam.Op.OPEN? openOffset: len - 1;
201        final LocatedBlocks locations = np.getBlockLocations(path, offset, 1);
202        final int count = locations.locatedBlockCount();
203        if (count > 0) {
204          return JspHelper.bestNode(locations.get(0).getLocations(), false, conf);
205        }
206      }
207    } 
208
209    return (DatanodeDescriptor)bm.getDatanodeManager().getNetworkTopology(
210        ).chooseRandom(NodeBase.ROOT);
211  }
212
213  private Token<? extends TokenIdentifier> generateDelegationToken(
214      final NameNode namenode, final UserGroupInformation ugi,
215      final String renewer) throws IOException {
216    final Credentials c = DelegationTokenSecretManager.createCredentials(
217        namenode, ugi, renewer != null? renewer: ugi.getShortUserName());
218    final Token<? extends TokenIdentifier> t = c.getAllTokens().iterator().next();
219    Text kind = request.getScheme().equals("http") ? WebHdfsFileSystem.TOKEN_KIND : SWebHdfsFileSystem.TOKEN_KIND;
220    t.setKind(kind);
221    return t;
222  }
223
224  private URI redirectURI(final NameNode namenode,
225      final UserGroupInformation ugi, final DelegationParam delegation,
226      final UserParam username, final DoAsParam doAsUser,
227      final String path, final HttpOpParam.Op op, final long openOffset,
228      final long blocksize,
229      final Param<?, ?>... parameters) throws URISyntaxException, IOException {
230    final Configuration conf = (Configuration)context.getAttribute(JspHelper.CURRENT_CONF);
231    final DatanodeInfo dn = chooseDatanode(namenode, path, op, openOffset,
232        blocksize, conf);
233
234    final String delegationQuery;
235    if (!UserGroupInformation.isSecurityEnabled()) {
236      //security disabled
237      delegationQuery = Param.toSortedString("&", doAsUser, username);
238    } else if (delegation.getValue() != null) {
239      //client has provided a token
240      delegationQuery = "&" + delegation;
241    } else {
242      //generate a token
243      final Token<? extends TokenIdentifier> t = generateDelegationToken(
244          namenode, ugi, request.getUserPrincipal().getName());
245      delegationQuery = "&" + new DelegationParam(t.encodeToUrlString());
246    }
247    final String query = op.toQueryString() + delegationQuery
248        + "&" + new NamenodeRpcAddressParam(namenode)
249        + Param.toSortedString("&", parameters);
250    final String uripath = WebHdfsFileSystem.PATH_PREFIX + path;
251
252    final String scheme = request.getScheme();
253    int port = "http".equals(scheme) ? dn.getInfoPort() : dn
254        .getInfoSecurePort();
255    final URI uri = new URI(scheme, null, dn.getHostName(), port, uripath,
256        query, null);
257
258    if (LOG.isTraceEnabled()) {
259      LOG.trace("redirectURI=" + uri);
260    }
261    return uri;
262  }
263
264  /** Handle HTTP PUT request for the root. */
265  @PUT
266  @Path("/")
267  @Consumes({"*/*"})
268  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
269  public Response putRoot(
270      @Context final UserGroupInformation ugi,
271      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
272          final DelegationParam delegation,
273      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
274          final UserParam username,
275      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
276          final DoAsParam doAsUser,
277      @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
278          final PutOpParam op,
279      @QueryParam(DestinationParam.NAME) @DefaultValue(DestinationParam.DEFAULT)
280          final DestinationParam destination,
281      @QueryParam(OwnerParam.NAME) @DefaultValue(OwnerParam.DEFAULT)
282          final OwnerParam owner,
283      @QueryParam(GroupParam.NAME) @DefaultValue(GroupParam.DEFAULT)
284          final GroupParam group,
285      @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
286          final PermissionParam permission,
287      @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
288          final OverwriteParam overwrite,
289      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
290          final BufferSizeParam bufferSize,
291      @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
292          final ReplicationParam replication,
293      @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
294          final BlockSizeParam blockSize,
295      @QueryParam(ModificationTimeParam.NAME) @DefaultValue(ModificationTimeParam.DEFAULT)
296          final ModificationTimeParam modificationTime,
297      @QueryParam(AccessTimeParam.NAME) @DefaultValue(AccessTimeParam.DEFAULT)
298          final AccessTimeParam accessTime,
299      @QueryParam(RenameOptionSetParam.NAME) @DefaultValue(RenameOptionSetParam.DEFAULT)
300          final RenameOptionSetParam renameOptions,
301      @QueryParam(CreateParentParam.NAME) @DefaultValue(CreateParentParam.DEFAULT)
302          final CreateParentParam createParent,
303      @QueryParam(TokenArgumentParam.NAME) @DefaultValue(TokenArgumentParam.DEFAULT)
304          final TokenArgumentParam delegationTokenArgument
305      ) throws IOException, InterruptedException {
306    return put(ugi, delegation, username, doAsUser, ROOT, op, destination,
307        owner, group, permission, overwrite, bufferSize, replication,
308        blockSize, modificationTime, accessTime, renameOptions, createParent,
309        delegationTokenArgument);
310  }
311
312  /** Handle HTTP PUT request. */
313  @PUT
314  @Path("{" + UriFsPathParam.NAME + ":.*}")
315  @Consumes({"*/*"})
316  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
317  public Response put(
318      @Context final UserGroupInformation ugi,
319      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
320          final DelegationParam delegation,
321      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
322          final UserParam username,
323      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
324          final DoAsParam doAsUser,
325      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
326      @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
327          final PutOpParam op,
328      @QueryParam(DestinationParam.NAME) @DefaultValue(DestinationParam.DEFAULT)
329          final DestinationParam destination,
330      @QueryParam(OwnerParam.NAME) @DefaultValue(OwnerParam.DEFAULT)
331          final OwnerParam owner,
332      @QueryParam(GroupParam.NAME) @DefaultValue(GroupParam.DEFAULT)
333          final GroupParam group,
334      @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
335          final PermissionParam permission,
336      @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
337          final OverwriteParam overwrite,
338      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
339          final BufferSizeParam bufferSize,
340      @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
341          final ReplicationParam replication,
342      @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
343          final BlockSizeParam blockSize,
344      @QueryParam(ModificationTimeParam.NAME) @DefaultValue(ModificationTimeParam.DEFAULT)
345          final ModificationTimeParam modificationTime,
346      @QueryParam(AccessTimeParam.NAME) @DefaultValue(AccessTimeParam.DEFAULT)
347          final AccessTimeParam accessTime,
348      @QueryParam(RenameOptionSetParam.NAME) @DefaultValue(RenameOptionSetParam.DEFAULT)
349          final RenameOptionSetParam renameOptions,
350      @QueryParam(CreateParentParam.NAME) @DefaultValue(CreateParentParam.DEFAULT)
351          final CreateParentParam createParent,
352      @QueryParam(TokenArgumentParam.NAME) @DefaultValue(TokenArgumentParam.DEFAULT)
353          final TokenArgumentParam delegationTokenArgument
354      ) throws IOException, InterruptedException {
355
356    init(ugi, delegation, username, doAsUser, path, op, destination, owner,
357        group, permission, overwrite, bufferSize, replication, blockSize,
358        modificationTime, accessTime, renameOptions, delegationTokenArgument);
359
360    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
361      @Override
362      public Response run() throws IOException, URISyntaxException {
363        REMOTE_ADDRESS.set(request.getRemoteAddr());
364        try {
365          return put(ugi, delegation, username, doAsUser,
366              path.getAbsolutePath(), op, destination, owner, group,
367              permission, overwrite, bufferSize, replication, blockSize,
368              modificationTime, accessTime, renameOptions, createParent,
369              delegationTokenArgument);
370        } finally {
371          REMOTE_ADDRESS.set(null);
372        }
373      }
374    });
375  }
376
377  private Response put(
378      final UserGroupInformation ugi,
379      final DelegationParam delegation,
380      final UserParam username,
381      final DoAsParam doAsUser,
382      final String fullpath,
383      final PutOpParam op,
384      final DestinationParam destination,
385      final OwnerParam owner,
386      final GroupParam group,
387      final PermissionParam permission,
388      final OverwriteParam overwrite,
389      final BufferSizeParam bufferSize,
390      final ReplicationParam replication,
391      final BlockSizeParam blockSize,
392      final ModificationTimeParam modificationTime,
393      final AccessTimeParam accessTime,
394      final RenameOptionSetParam renameOptions,
395      final CreateParentParam createParent,
396      final TokenArgumentParam delegationTokenArgument
397      ) throws IOException, URISyntaxException {
398
399    final Configuration conf = (Configuration)context.getAttribute(JspHelper.CURRENT_CONF);
400    final NameNode namenode = (NameNode)context.getAttribute("name.node");
401    final NamenodeProtocols np = namenode.getRpcServer();
402
403    switch(op.getValue()) {
404    case CREATE:
405    {
406      final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser,
407          fullpath, op.getValue(), -1L, blockSize.getValue(conf),
408          permission, overwrite, bufferSize, replication, blockSize);
409      return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
410    } 
411    case MKDIRS:
412    {
413      final boolean b = np.mkdirs(fullpath, permission.getFsPermission(), true);
414      final String js = JsonUtil.toJsonString("boolean", b);
415      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
416    }
417    case CREATESYMLINK:
418    {
419      np.createSymlink(destination.getValue(), fullpath,
420          PermissionParam.getDefaultFsPermission(), createParent.getValue());
421      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
422    }
423    case RENAME:
424    {
425      final EnumSet<Options.Rename> s = renameOptions.getValue();
426      if (s.isEmpty()) {
427        final boolean b = np.rename(fullpath, destination.getValue());
428        final String js = JsonUtil.toJsonString("boolean", b);
429        return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
430      } else {
431        np.rename2(fullpath, destination.getValue(),
432            s.toArray(new Options.Rename[s.size()]));
433        return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
434      }
435    }
436    case SETREPLICATION:
437    {
438      final boolean b = np.setReplication(fullpath, replication.getValue(conf));
439      final String js = JsonUtil.toJsonString("boolean", b);
440      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
441    }
442    case SETOWNER:
443    {
444      if (owner.getValue() == null && group.getValue() == null) {
445        throw new IllegalArgumentException("Both owner and group are empty.");
446      }
447
448      np.setOwner(fullpath, owner.getValue(), group.getValue());
449      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
450    }
451    case SETPERMISSION:
452    {
453      np.setPermission(fullpath, permission.getFsPermission());
454      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
455    }
456    case SETTIMES:
457    {
458      np.setTimes(fullpath, modificationTime.getValue(), accessTime.getValue());
459      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
460    }
461    case RENEWDELEGATIONTOKEN:
462    {
463      final Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>();
464      token.decodeFromUrlString(delegationTokenArgument.getValue());
465      final long expiryTime = np.renewDelegationToken(token);
466      final String js = JsonUtil.toJsonString("long", expiryTime);
467      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
468    }
469    case CANCELDELEGATIONTOKEN:
470    {
471      final Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>();
472      token.decodeFromUrlString(delegationTokenArgument.getValue());
473      np.cancelDelegationToken(token);
474      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
475    }
476    default:
477      throw new UnsupportedOperationException(op + " is not supported");
478    }
479  }
480
481  /** Handle HTTP POST request for the root. */
482  @POST
483  @Path("/")
484  @Consumes({"*/*"})
485  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
486  public Response postRoot(
487      @Context final UserGroupInformation ugi,
488      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
489          final DelegationParam delegation,
490      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
491          final UserParam username,
492      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
493          final DoAsParam doAsUser,
494      @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
495          final PostOpParam op,
496      @QueryParam(ConcatSourcesParam.NAME) @DefaultValue(ConcatSourcesParam.DEFAULT)
497          final ConcatSourcesParam concatSrcs,
498      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
499          final BufferSizeParam bufferSize
500      ) throws IOException, InterruptedException {
501    return post(ugi, delegation, username, doAsUser, ROOT, op, concatSrcs, bufferSize);
502  }
503
504  /** Handle HTTP POST request. */
505  @POST
506  @Path("{" + UriFsPathParam.NAME + ":.*}")
507  @Consumes({"*/*"})
508  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
509  public Response post(
510      @Context final UserGroupInformation ugi,
511      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
512          final DelegationParam delegation,
513      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
514          final UserParam username,
515      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
516          final DoAsParam doAsUser,
517      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
518      @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
519          final PostOpParam op,
520      @QueryParam(ConcatSourcesParam.NAME) @DefaultValue(ConcatSourcesParam.DEFAULT)
521          final ConcatSourcesParam concatSrcs,
522      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
523          final BufferSizeParam bufferSize
524      ) throws IOException, InterruptedException {
525
526    init(ugi, delegation, username, doAsUser, path, op, concatSrcs, bufferSize);
527
528    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
529      @Override
530      public Response run() throws IOException, URISyntaxException {
531        REMOTE_ADDRESS.set(request.getRemoteAddr());
532        try {
533          return post(ugi, delegation, username, doAsUser,
534              path.getAbsolutePath(), op, concatSrcs, bufferSize);
535        } finally {
536          REMOTE_ADDRESS.set(null);
537        }
538      }
539    });
540  }
541
542  private Response post(
543      final UserGroupInformation ugi,
544      final DelegationParam delegation,
545      final UserParam username,
546      final DoAsParam doAsUser,
547      final String fullpath,
548      final PostOpParam op,
549      final ConcatSourcesParam concatSrcs,
550      final BufferSizeParam bufferSize
551      ) throws IOException, URISyntaxException {
552    final NameNode namenode = (NameNode)context.getAttribute("name.node");
553
554    switch(op.getValue()) {
555    case APPEND:
556    {
557      final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser,
558          fullpath, op.getValue(), -1L, -1L, bufferSize);
559      return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
560    }
561    case CONCAT:
562    {
563      namenode.getRpcServer().concat(fullpath, concatSrcs.getAbsolutePaths());
564      return Response.ok().build();
565    }
566    default:
567      throw new UnsupportedOperationException(op + " is not supported");
568    }
569  }
570
571  /** Handle HTTP GET request for the root. */
572  @GET
573  @Path("/")
574  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
575  public Response getRoot(
576      @Context final UserGroupInformation ugi,
577      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
578          final DelegationParam delegation,
579      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
580          final UserParam username,
581      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
582          final DoAsParam doAsUser,
583      @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
584          final GetOpParam op,
585      @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
586          final OffsetParam offset,
587      @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
588          final LengthParam length,
589      @QueryParam(RenewerParam.NAME) @DefaultValue(RenewerParam.DEFAULT)
590          final RenewerParam renewer,
591      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
592          final BufferSizeParam bufferSize
593      ) throws IOException, InterruptedException {
594    return get(ugi, delegation, username, doAsUser, ROOT, op,
595        offset, length, renewer, bufferSize);
596  }
597
598  /** Handle HTTP GET request. */
599  @GET
600  @Path("{" + UriFsPathParam.NAME + ":.*}")
601  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
602  public Response get(
603      @Context final UserGroupInformation ugi,
604      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
605          final DelegationParam delegation,
606      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
607          final UserParam username,
608      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
609          final DoAsParam doAsUser,
610      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
611      @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
612          final GetOpParam op,
613      @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
614          final OffsetParam offset,
615      @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
616          final LengthParam length,
617      @QueryParam(RenewerParam.NAME) @DefaultValue(RenewerParam.DEFAULT)
618          final RenewerParam renewer,
619      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
620          final BufferSizeParam bufferSize
621      ) throws IOException, InterruptedException {
622
623    init(ugi, delegation, username, doAsUser, path, op,
624        offset, length, renewer, bufferSize);
625
626    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
627      @Override
628      public Response run() throws IOException, URISyntaxException {
629        REMOTE_ADDRESS.set(request.getRemoteAddr());
630        try {
631          return get(ugi, delegation, username, doAsUser,
632              path.getAbsolutePath(), op, offset, length, renewer, bufferSize);
633        } finally {
634          REMOTE_ADDRESS.set(null);
635        }
636      }
637    });
638  }
639
640  private Response get(
641      final UserGroupInformation ugi,
642      final DelegationParam delegation,
643      final UserParam username,
644      final DoAsParam doAsUser,
645      final String fullpath,
646      final GetOpParam op,
647      final OffsetParam offset,
648      final LengthParam length,
649      final RenewerParam renewer,
650      final BufferSizeParam bufferSize
651      ) throws IOException, URISyntaxException {
652    final NameNode namenode = (NameNode)context.getAttribute("name.node");
653    final NamenodeProtocols np = namenode.getRpcServer();
654
655    switch(op.getValue()) {
656    case OPEN:
657    {
658      final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser,
659          fullpath, op.getValue(), offset.getValue(), -1L, offset, length, bufferSize);
660      return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
661    }
662    case GET_BLOCK_LOCATIONS:
663    {
664      final long offsetValue = offset.getValue();
665      final Long lengthValue = length.getValue();
666      final LocatedBlocks locatedblocks = np.getBlockLocations(fullpath,
667          offsetValue, lengthValue != null? lengthValue: Long.MAX_VALUE);
668      final String js = JsonUtil.toJsonString(locatedblocks);
669      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
670    }
671    case GETFILESTATUS:
672    {
673      final HdfsFileStatus status = np.getFileInfo(fullpath);
674      if (status == null) {
675        throw new FileNotFoundException("File does not exist: " + fullpath);
676      }
677
678      final String js = JsonUtil.toJsonString(status, true);
679      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
680    }
681    case LISTSTATUS:
682    {
683      final StreamingOutput streaming = getListingStream(np, fullpath);
684      return Response.ok(streaming).type(MediaType.APPLICATION_JSON).build();
685    }
686    case GETCONTENTSUMMARY:
687    {
688      final ContentSummary contentsummary = np.getContentSummary(fullpath);
689      final String js = JsonUtil.toJsonString(contentsummary);
690      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
691    }
692    case GETFILECHECKSUM:
693    {
694      final URI uri = redirectURI(namenode, ugi, delegation, username, doAsUser,
695          fullpath, op.getValue(), -1L, -1L);
696      return Response.temporaryRedirect(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
697    }
698    case GETDELEGATIONTOKEN:
699    {
700      if (delegation.getValue() != null) {
701        throw new IllegalArgumentException(delegation.getName()
702            + " parameter is not null.");
703      }
704      final Token<? extends TokenIdentifier> token = generateDelegationToken(
705          namenode, ugi, renewer.getValue());
706      final String js = JsonUtil.toJsonString(token);
707      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
708    }
709    case GETHOMEDIRECTORY:
710    {
711      final String js = JsonUtil.toJsonString(
712          org.apache.hadoop.fs.Path.class.getSimpleName(),
713          WebHdfsFileSystem.getHomeDirectoryString(ugi));
714      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
715    }
716    default:
717      throw new UnsupportedOperationException(op + " is not supported");
718    }
719  }
720
721  private static DirectoryListing getDirectoryListing(final NamenodeProtocols np,
722      final String p, byte[] startAfter) throws IOException {
723    final DirectoryListing listing = np.getListing(p, startAfter, false);
724    if (listing == null) { // the directory does not exist
725      throw new FileNotFoundException("File " + p + " does not exist.");
726    }
727    return listing;
728  }
729  
730  private static StreamingOutput getListingStream(final NamenodeProtocols np, 
731      final String p) throws IOException {
732    // allows exceptions like FNF or ACE to prevent http response of 200 for
733    // a failure since we can't (currently) return error responses in the
734    // middle of a streaming operation
735    final DirectoryListing firstDirList = getDirectoryListing(np, p,
736        HdfsFileStatus.EMPTY_NAME);
737
738    // must save ugi because the streaming object will be executed outside
739    // the remote user's ugi
740    final UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
741    return new StreamingOutput() {
742      @Override
743      public void write(final OutputStream outstream) throws IOException {
744        final PrintWriter out = new PrintWriter(new OutputStreamWriter(
745            outstream, Charsets.UTF_8));
746        out.println("{\"" + FileStatus.class.getSimpleName() + "es\":{\""
747            + FileStatus.class.getSimpleName() + "\":[");
748
749        try {
750          // restore remote user's ugi
751          ugi.doAs(new PrivilegedExceptionAction<Void>() {
752            @Override
753            public Void run() throws IOException {
754              long n = 0;
755              for (DirectoryListing dirList = firstDirList; ;
756                   dirList = getDirectoryListing(np, p, dirList.getLastName())
757              ) {
758                // send each segment of the directory listing
759                for (HdfsFileStatus s : dirList.getPartialListing()) {
760                  if (n++ > 0) {
761                    out.println(',');
762                  }
763                  out.print(JsonUtil.toJsonString(s, false));
764                }
765                // stop if last segment
766                if (!dirList.hasMore()) {
767                  break;
768                }
769              }
770              return null;
771            }
772          });
773        } catch (InterruptedException e) {
774          throw new IOException(e);
775        }
776        
777        out.println();
778        out.println("]}}");
779        out.flush();
780      }
781    };
782  }
783
784  /** Handle HTTP DELETE request for the root. */
785  @DELETE
786  @Path("/")
787  @Produces(MediaType.APPLICATION_JSON)
788  public Response deleteRoot(
789      @Context final UserGroupInformation ugi,
790      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
791          final DelegationParam delegation,
792      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
793          final UserParam username,
794      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
795          final DoAsParam doAsUser,
796      @QueryParam(DeleteOpParam.NAME) @DefaultValue(DeleteOpParam.DEFAULT)
797          final DeleteOpParam op,
798      @QueryParam(RecursiveParam.NAME) @DefaultValue(RecursiveParam.DEFAULT)
799          final RecursiveParam recursive
800      ) throws IOException, InterruptedException {
801    return delete(ugi, delegation, username, doAsUser, ROOT, op, recursive);
802  }
803
804  /** Handle HTTP DELETE request. */
805  @DELETE
806  @Path("{" + UriFsPathParam.NAME + ":.*}")
807  @Produces(MediaType.APPLICATION_JSON)
808  public Response delete(
809      @Context final UserGroupInformation ugi,
810      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
811          final DelegationParam delegation,
812      @QueryParam(UserParam.NAME) @DefaultValue(UserParam.DEFAULT)
813          final UserParam username,
814      @QueryParam(DoAsParam.NAME) @DefaultValue(DoAsParam.DEFAULT)
815          final DoAsParam doAsUser,
816      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
817      @QueryParam(DeleteOpParam.NAME) @DefaultValue(DeleteOpParam.DEFAULT)
818          final DeleteOpParam op,
819      @QueryParam(RecursiveParam.NAME) @DefaultValue(RecursiveParam.DEFAULT)
820          final RecursiveParam recursive
821      ) throws IOException, InterruptedException {
822
823    init(ugi, delegation, username, doAsUser, path, op, recursive);
824
825    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
826      @Override
827      public Response run() throws IOException {
828        REMOTE_ADDRESS.set(request.getRemoteAddr());
829        try {
830          return delete(ugi, delegation, username, doAsUser,
831              path.getAbsolutePath(), op, recursive);
832        } finally {
833          REMOTE_ADDRESS.set(null);
834        }
835      }
836    });
837  }
838
839  private Response delete(
840      final UserGroupInformation ugi,
841      final DelegationParam delegation,
842      final UserParam username,
843      final DoAsParam doAsUser,
844      final String fullpath,
845      final DeleteOpParam op,
846      final RecursiveParam recursive
847      ) throws IOException {
848    final NameNode namenode = (NameNode)context.getAttribute("name.node");
849
850    switch(op.getValue()) {
851    case DELETE:
852    {
853      final boolean b = namenode.getRpcServer().delete(fullpath, recursive.getValue());
854      final String js = JsonUtil.toJsonString("boolean", b);
855      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
856    }
857    default:
858      throw new UnsupportedOperationException(op + " is not supported");
859    }
860  }
861}