001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs.server.datanode.web.resources;
019
020import java.io.IOException;
021import java.io.InputStream;
022import java.net.InetSocketAddress;
023import java.net.URI;
024import java.net.URISyntaxException;
025import java.security.PrivilegedExceptionAction;
026import java.util.EnumSet;
027
028import javax.servlet.ServletContext;
029import javax.servlet.http.HttpServletResponse;
030import javax.ws.rs.Consumes;
031import javax.ws.rs.DefaultValue;
032import javax.ws.rs.GET;
033import javax.ws.rs.POST;
034import javax.ws.rs.PUT;
035import javax.ws.rs.Path;
036import javax.ws.rs.PathParam;
037import javax.ws.rs.Produces;
038import javax.ws.rs.QueryParam;
039import javax.ws.rs.core.Context;
040import javax.ws.rs.core.MediaType;
041import javax.ws.rs.core.Response;
042
043import org.apache.commons.logging.Log;
044import org.apache.commons.logging.LogFactory;
045import org.apache.hadoop.conf.Configuration;
046import org.apache.hadoop.fs.CreateFlag;
047import org.apache.hadoop.fs.FSDataOutputStream;
048import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
049import org.apache.hadoop.fs.permission.FsPermission;
050import org.apache.hadoop.hdfs.DFSClient;
051import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
052import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
053import org.apache.hadoop.hdfs.server.datanode.DataNode;
054import org.apache.hadoop.hdfs.server.namenode.NameNode;
055import org.apache.hadoop.hdfs.web.JsonUtil;
056import org.apache.hadoop.hdfs.web.ParamFilter;
057import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
058import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
059import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
060import org.apache.hadoop.hdfs.web.resources.DelegationParam;
061import org.apache.hadoop.hdfs.web.resources.GetOpParam;
062import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
063import org.apache.hadoop.hdfs.web.resources.LengthParam;
064import org.apache.hadoop.hdfs.web.resources.NamenodeRpcAddressParam;
065import org.apache.hadoop.hdfs.web.resources.OffsetParam;
066import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
067import org.apache.hadoop.hdfs.web.resources.Param;
068import org.apache.hadoop.hdfs.web.resources.PermissionParam;
069import org.apache.hadoop.hdfs.web.resources.PostOpParam;
070import org.apache.hadoop.hdfs.web.resources.PutOpParam;
071import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
072import org.apache.hadoop.hdfs.web.resources.UriFsPathParam;
073import org.apache.hadoop.io.IOUtils;
074import org.apache.hadoop.security.SecurityUtil;
075import org.apache.hadoop.security.UserGroupInformation;
076import org.apache.hadoop.security.token.Token;
077
078import com.sun.jersey.spi.container.ResourceFilters;
079
080/** Web-hdfs DataNode implementation. */
081@Path("")
082@ResourceFilters(ParamFilter.class)
083public class DatanodeWebHdfsMethods {
084  public static final Log LOG = LogFactory.getLog(DatanodeWebHdfsMethods.class);
085
086  private static final UriFsPathParam ROOT = new UriFsPathParam("");
087
088  private @Context ServletContext context;
089  private @Context HttpServletResponse response;
090
091  private void init(final UserGroupInformation ugi,
092      final DelegationParam delegation, final InetSocketAddress nnRpcAddr,
093      final UriFsPathParam path, final HttpOpParam<?> op,
094      final Param<?, ?>... parameters) throws IOException {
095    if (LOG.isTraceEnabled()) {
096      LOG.trace("HTTP " + op.getValue().getType() + ": " + op + ", " + path
097          + ", ugi=" + ugi + Param.toSortedString(", ", parameters));
098    }
099    if (nnRpcAddr == null) {
100      throw new IllegalArgumentException(NamenodeRpcAddressParam.NAME
101          + " is not specified.");
102    }
103
104    //clear content type
105    response.setContentType(null);
106    
107    if (UserGroupInformation.isSecurityEnabled()) {
108      //add a token for RPC.
109      final Token<DelegationTokenIdentifier> token = 
110          new Token<DelegationTokenIdentifier>();
111      token.decodeFromUrlString(delegation.getValue());
112      SecurityUtil.setTokenService(token, nnRpcAddr);
113      token.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
114      ugi.addToken(token);
115    }
116  }
117
118  /** Handle HTTP PUT request for the root. */
119  @PUT
120  @Path("/")
121  @Consumes({"*/*"})
122  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
123  public Response putRoot(
124      final InputStream in,
125      @Context final UserGroupInformation ugi,
126      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
127          final DelegationParam delegation,
128      @QueryParam(NamenodeRpcAddressParam.NAME) 
129      @DefaultValue(NamenodeRpcAddressParam.DEFAULT) 
130          final NamenodeRpcAddressParam namenodeRpcAddress,
131      @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
132          final PutOpParam op,
133      @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
134          final PermissionParam permission,
135      @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
136          final OverwriteParam overwrite,
137      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
138          final BufferSizeParam bufferSize,
139      @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
140          final ReplicationParam replication,
141      @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
142          final BlockSizeParam blockSize
143      ) throws IOException, InterruptedException {
144    return put(in, ugi, delegation, namenodeRpcAddress, ROOT, op, permission,
145        overwrite, bufferSize, replication, blockSize);
146  }
147
148  /** Handle HTTP PUT request. */
149  @PUT
150  @Path("{" + UriFsPathParam.NAME + ":.*}")
151  @Consumes({"*/*"})
152  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
153  public Response put(
154      final InputStream in,
155      @Context final UserGroupInformation ugi,
156      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
157          final DelegationParam delegation,
158      @QueryParam(NamenodeRpcAddressParam.NAME)
159      @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
160          final NamenodeRpcAddressParam namenodeRpcAddress,
161      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
162      @QueryParam(PutOpParam.NAME) @DefaultValue(PutOpParam.DEFAULT)
163          final PutOpParam op,
164      @QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
165          final PermissionParam permission,
166      @QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
167          final OverwriteParam overwrite,
168      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
169          final BufferSizeParam bufferSize,
170      @QueryParam(ReplicationParam.NAME) @DefaultValue(ReplicationParam.DEFAULT)
171          final ReplicationParam replication,
172      @QueryParam(BlockSizeParam.NAME) @DefaultValue(BlockSizeParam.DEFAULT)
173          final BlockSizeParam blockSize
174      ) throws IOException, InterruptedException {
175
176    final InetSocketAddress nnRpcAddr = namenodeRpcAddress.getValue();
177    init(ugi, delegation, nnRpcAddr, path, op, permission,
178        overwrite, bufferSize, replication, blockSize);
179
180    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
181      @Override
182      public Response run() throws IOException, URISyntaxException {
183        return put(in, ugi, delegation, nnRpcAddr, path.getAbsolutePath(), op,
184            permission, overwrite, bufferSize, replication, blockSize);
185      }
186    });
187  }
188
189  private Response put(
190      final InputStream in,
191      final UserGroupInformation ugi,
192      final DelegationParam delegation,
193      final InetSocketAddress nnRpcAddr,
194      final String fullpath,
195      final PutOpParam op,
196      final PermissionParam permission,
197      final OverwriteParam overwrite,
198      final BufferSizeParam bufferSize,
199      final ReplicationParam replication,
200      final BlockSizeParam blockSize
201      ) throws IOException, URISyntaxException {
202    final DataNode datanode = (DataNode)context.getAttribute("datanode");
203
204    switch(op.getValue()) {
205    case CREATE:
206    {
207      final Configuration conf = new Configuration(datanode.getConf());
208      conf.set(FsPermission.UMASK_LABEL, "000");
209
210      final int b = bufferSize.getValue(conf);
211      DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
212      FSDataOutputStream out = null;
213      try {
214        out = new FSDataOutputStream(dfsclient.create(
215            fullpath, permission.getFsPermission(), 
216            overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE)
217                : EnumSet.of(CreateFlag.CREATE),
218            replication.getValue(conf), blockSize.getValue(conf), null, b, null), null);
219        IOUtils.copyBytes(in, out, b);
220        out.close();
221        out = null;
222        dfsclient.close();
223        dfsclient = null;
224      } finally {
225        IOUtils.cleanup(LOG, out);
226        IOUtils.cleanup(LOG, dfsclient);
227      }
228      final InetSocketAddress nnHttpAddr = NameNode.getHttpAddress(conf);
229      final URI uri = new URI(WebHdfsFileSystem.SCHEME, null,
230          nnHttpAddr.getHostName(), nnHttpAddr.getPort(), fullpath, null, null);
231      return Response.created(uri).type(MediaType.APPLICATION_OCTET_STREAM).build();
232    }
233    default:
234      throw new UnsupportedOperationException(op + " is not supported");
235    }
236  }
237
238  /** Handle HTTP POST request for the root for the root. */
239  @POST
240  @Path("/")
241  @Consumes({"*/*"})
242  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
243  public Response postRoot(
244      final InputStream in,
245      @Context final UserGroupInformation ugi,
246      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
247          final DelegationParam delegation,
248      @QueryParam(NamenodeRpcAddressParam.NAME)
249      @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
250          final NamenodeRpcAddressParam namenodeRpcAddress,
251      @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
252          final PostOpParam op,
253      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
254          final BufferSizeParam bufferSize
255      ) throws IOException, InterruptedException {
256    return post(in, ugi, delegation, namenodeRpcAddress, ROOT, op, bufferSize);
257  }
258
259  /** Handle HTTP POST request. */
260  @POST
261  @Path("{" + UriFsPathParam.NAME + ":.*}")
262  @Consumes({"*/*"})
263  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
264  public Response post(
265      final InputStream in,
266      @Context final UserGroupInformation ugi,
267      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
268          final DelegationParam delegation,
269      @QueryParam(NamenodeRpcAddressParam.NAME)
270      @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
271          final NamenodeRpcAddressParam namenodeRpcAddress,
272      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
273      @QueryParam(PostOpParam.NAME) @DefaultValue(PostOpParam.DEFAULT)
274          final PostOpParam op,
275      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
276          final BufferSizeParam bufferSize
277      ) throws IOException, InterruptedException {
278
279    final InetSocketAddress nnRpcAddr = namenodeRpcAddress.getValue();
280    init(ugi, delegation, nnRpcAddr, path, op, bufferSize);
281
282    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
283      @Override
284      public Response run() throws IOException {
285        return post(in, ugi, delegation, nnRpcAddr, path.getAbsolutePath(), op,
286            bufferSize);
287      }
288    });
289  }
290
291  private Response post(
292      final InputStream in,
293      final UserGroupInformation ugi,
294      final DelegationParam delegation,
295      final InetSocketAddress nnRpcAddr,
296      final String fullpath,
297      final PostOpParam op,
298      final BufferSizeParam bufferSize
299      ) throws IOException {
300    final DataNode datanode = (DataNode)context.getAttribute("datanode");
301
302    switch(op.getValue()) {
303    case APPEND:
304    {
305      final Configuration conf = new Configuration(datanode.getConf());
306      final int b = bufferSize.getValue(conf);
307      DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
308      FSDataOutputStream out = null;
309      try {
310        out = dfsclient.append(fullpath, b, null, null);
311        IOUtils.copyBytes(in, out, b);
312        out.close();
313        out = null;
314        dfsclient.close();
315        dfsclient = null;
316      } finally {
317        IOUtils.cleanup(LOG, out);
318        IOUtils.cleanup(LOG, dfsclient);
319      }
320      return Response.ok().type(MediaType.APPLICATION_OCTET_STREAM).build();
321    }
322    default:
323      throw new UnsupportedOperationException(op + " is not supported");
324    }
325  }
326
327  /** Handle HTTP GET request for the root. */
328  @GET
329  @Path("/")
330  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
331  public Response getRoot(
332      @Context final UserGroupInformation ugi,
333      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
334          final DelegationParam delegation,
335      @QueryParam(NamenodeRpcAddressParam.NAME)
336      @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
337          final NamenodeRpcAddressParam namenodeRpcAddress,
338      @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
339          final GetOpParam op,
340      @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
341          final OffsetParam offset,
342      @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
343          final LengthParam length,
344      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
345          final BufferSizeParam bufferSize
346      ) throws IOException, InterruptedException {
347    return get(ugi, delegation, namenodeRpcAddress, ROOT, op, offset, length,
348        bufferSize);
349  }
350
351  /** Handle HTTP GET request. */
352  @GET
353  @Path("{" + UriFsPathParam.NAME + ":.*}")
354  @Produces({MediaType.APPLICATION_OCTET_STREAM, MediaType.APPLICATION_JSON})
355  public Response get(
356      @Context final UserGroupInformation ugi,
357      @QueryParam(DelegationParam.NAME) @DefaultValue(DelegationParam.DEFAULT)
358          final DelegationParam delegation,
359      @QueryParam(NamenodeRpcAddressParam.NAME)
360      @DefaultValue(NamenodeRpcAddressParam.DEFAULT)
361          final NamenodeRpcAddressParam namenodeRpcAddress,
362      @PathParam(UriFsPathParam.NAME) final UriFsPathParam path,
363      @QueryParam(GetOpParam.NAME) @DefaultValue(GetOpParam.DEFAULT)
364          final GetOpParam op,
365      @QueryParam(OffsetParam.NAME) @DefaultValue(OffsetParam.DEFAULT)
366          final OffsetParam offset,
367      @QueryParam(LengthParam.NAME) @DefaultValue(LengthParam.DEFAULT)
368          final LengthParam length,
369      @QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
370          final BufferSizeParam bufferSize
371      ) throws IOException, InterruptedException {
372
373    final InetSocketAddress nnRpcAddr = namenodeRpcAddress.getValue();
374    init(ugi, delegation, nnRpcAddr, path, op, offset, length, bufferSize);
375
376    return ugi.doAs(new PrivilegedExceptionAction<Response>() {
377      @Override
378      public Response run() throws IOException {
379        return get(ugi, delegation, nnRpcAddr, path.getAbsolutePath(), op,
380            offset, length, bufferSize);
381      }
382    });
383  }
384
385  private Response get(
386      final UserGroupInformation ugi,
387      final DelegationParam delegation,
388      final InetSocketAddress nnRpcAddr,
389      final String fullpath,
390      final GetOpParam op,
391      final OffsetParam offset,
392      final LengthParam length,
393      final BufferSizeParam bufferSize
394      ) throws IOException {
395    final DataNode datanode = (DataNode)context.getAttribute("datanode");
396    final Configuration conf = new Configuration(datanode.getConf());
397
398    switch(op.getValue()) {
399    case OPEN:
400    {
401      final int b = bufferSize.getValue(conf);
402      final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
403      HdfsDataInputStream in = null;
404      try {
405        in = new HdfsDataInputStream(dfsclient.open(fullpath, b, true));
406        in.seek(offset.getValue());
407      } catch(IOException ioe) {
408        IOUtils.cleanup(LOG, in);
409        IOUtils.cleanup(LOG, dfsclient);
410        throw ioe;
411      }
412      
413      final long n = length.getValue() != null ?
414        Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) :
415        in.getVisibleLength() - offset.getValue();
416
417      /**
418       * Allow the Web UI to perform an AJAX request to get the data.
419       */
420      return Response.ok(new OpenEntity(in, n, dfsclient))
421          .type(MediaType.APPLICATION_OCTET_STREAM)
422          .header("Access-Control-Allow-Methods", "GET")
423          .header("Access-Control-Allow-Origin", "*")
424          .build();
425    }
426    case GETFILECHECKSUM:
427    {
428      MD5MD5CRC32FileChecksum checksum = null;
429      DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
430      try {
431        checksum = dfsclient.getFileChecksum(fullpath);
432        dfsclient.close();
433        dfsclient = null;
434      } finally {
435        IOUtils.cleanup(LOG, dfsclient);
436      }
437      final String js = JsonUtil.toJsonString(checksum);
438      return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
439    }
440    default:
441      throw new UnsupportedOperationException(op + " is not supported");
442    }
443  }
444}