yarn logs -applicationId命令java版本简单实现 

  1 import java.io.DataInputStream;
  2 import java.io.EOFException;
  3 import java.io.FileNotFoundException;
  4 import java.io.PrintStream;
  5 
  6 import org.apache.commons.lang.StringUtils;
  7 import org.apache.hadoop.conf.Configuration;
  8 import org.apache.hadoop.fs.FileContext;
  9 import org.apache.hadoop.fs.FileStatus;
 10 import org.apache.hadoop.fs.Path;
 11 import org.apache.hadoop.fs.RemoteIterator;
 12 import org.apache.hadoop.security.UserGroupInformation;
 13 import org.apache.hadoop.yarn.api.records.ApplicationId;
 14 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 15 import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;
 16 import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
 17 import org.apache.hadoop.yarn.util.ConverterUtils;
 18 
 19 public class GetYarnLog {
 20     public static void main(String[] args) {
 21         run("application_1535700682133_0496");
 22     }
 23     
 24     public static int run(String appIdStr) throws Throwable{
 25  
 26     
 27          Configuration conf = new YarnConfiguration();
 28          conf.addResource(new Path("/etc/hadoop/conf.cloudera.yarn/core-site.xml"));
 29          conf.addResource(new Path("/etc/hadoop/conf.cloudera.yarn/yarn-site.xml"));
 30          conf.addResource(new Path("/etc/hadoop/conf.cloudera.yarn/hdfs-site.xml"));
 31          if(appIdStr == null || appIdStr.equals(""))
 32           {
 33              System.out.println("appId is null!");
 34              return -1;
 35           }
 36          PrintStream out=new PrintStream(appIdStr); 
 37          ApplicationId appId = null;
 38          appId = ConverterUtils.toApplicationId(appIdStr);
 39          
 40          Path remoteRootLogDir = new Path(conf.get("yarn.nodemanager.remote-app-log-dir", "/tmp/logs"));
 41 
 42          String user =  UserGroupInformation.getCurrentUser().getShortUserName();;
 43          String logDirSuffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf);
 44          
 45          Path remoteAppLogDir = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir, appId, user, logDirSuffix);
 46          RemoteIterator<FileStatus> nodeFiles;
 47          try
 48          {
 49            Path qualifiedLogDir = FileContext.getFileContext(conf).makeQualified(remoteAppLogDir);
 50            nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), conf).listStatus(remoteAppLogDir);
 51          }
 52          catch (FileNotFoundException fnf)
 53          {
 54            logDirNotExist(remoteAppLogDir.toString());
 55            return -1;
 56          }
 57          
 58          boolean foundAnyLogs = false;
 59          while (nodeFiles.hasNext())
 60          {
 61            FileStatus thisNodeFile = (FileStatus)nodeFiles.next();
 62            if (!thisNodeFile.getPath().getName().endsWith(".tmp"))
 63            {
 64                System.out.println("NodeFileName = "+thisNodeFile.getPath().getName());
 65              AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(conf, thisNodeFile.getPath());
 66              try
 67              {
 68                AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey();
 69                DataInputStream valueStream = reader.next(key);
 70                for (;;)
 71                {
 72                  if (valueStream != null)
 73                  {
 74                    String containerString = "\n\nContainer: " + key + " on " + thisNodeFile.getPath().getName();
 75                    
 76                    out.println(containerString);
 77                    out.println(StringUtils.repeat("=", containerString.length()));
 78                    try
 79                    {
 80                      for (;;)
 81                      {
 82                        AggregatedLogFormat.LogReader.readAContainerLogsForALogType(valueStream, out, thisNodeFile.getModificationTime());
 83                        
 84                        foundAnyLogs = true;
 85                      }
 86                          
 87                    }
 88                    catch (EOFException eof)
 89                    {
 90                      key = new AggregatedLogFormat.LogKey();
 91                      valueStream = reader.next(key);
 92                       
 93                    }
 94                    
 95                  }else{
 96                      break;
 97                  }
 98                }
 99              }
100              finally
101              {
102                reader.close();
103              }
104            }
105          }
106          if (!foundAnyLogs)
107          {
108            emptyLogDir(remoteAppLogDir.toString());
109            return -1;
110          }
111          return 0;
112        }
113 }

 

posted @ 2018-09-12 16:28  一直爬行的蜗牛牛  阅读(3610)  评论(1)    收藏  举报