0


Hadoop-Yarn-NodeManager如何计算Linux系统上的资源信息

一、上下文

<Hadoop-Yarn-NodeManager都做了什么>中讲节点资源监控服务(NodeResourceMonitorImpl)时只是提了下SysInfoLinux,下面我们展开讲下

SysInfoLinux是用于计算Linux系统上的资源信息的插件

二、SysInfoLinux源码

package org.apache.hadoop.util;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStreamReader;
import java.io.IOException;
import java.math.BigInteger;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.HashSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * 用于计算Linux系统上的资源信息的插件
 */
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class SysInfoLinux extends SysInfo {
  private static final Logger LOG =
      LoggerFactory.getLogger(SysInfoLinux.class);

  /**
   * proc的meminfo虚拟文件具有以下格式的键值
   * "key:[ \t]*value[ \t]kB".
   */
  private static final String PROCFS_MEMFILE = "/proc/meminfo";
  private static final Pattern PROCFS_MEMFILE_FORMAT =
      Pattern.compile("^([a-zA-Z_()]*):[ \t]*([0-9]*)[ \t]*(kB)?");

  //我们需要meminfo中以下键的值
  private static final String MEMTOTAL_STRING = "MemTotal";
  private static final String SWAPTOTAL_STRING = "SwapTotal";
  private static final String MEMFREE_STRING = "MemFree";
  private static final String SWAPFREE_STRING = "SwapFree";
  private static final String INACTIVE_STRING = "Inactive";
  private static final String INACTIVEFILE_STRING = "Inactive(file)";
  private static final String HARDWARECORRUPTED_STRING = "HardwareCorrupted";
  private static final String HUGEPAGESTOTAL_STRING = "HugePages_Total";
  private static final String HUGEPAGESIZE_STRING = "Hugepagesize";

  /**
   * 解析/proc/cpuinfo的样板
   */
  private static final String PROCFS_CPUINFO = "/proc/cpuinfo";
  private static final Pattern PROCESSOR_FORMAT =
      Pattern.compile("^processor[ \t]:[ \t]*([0-9]*)");
  private static final Pattern FREQUENCY_FORMAT =
      Pattern.compile("^cpu MHz[ \t]*:[ \t]*([0-9.]*)");
  private static final Pattern PHYSICAL_ID_FORMAT =
      Pattern.compile("^physical id[ \t]*:[ \t]*([0-9]*)");
  private static final Pattern CORE_ID_FORMAT =
      Pattern.compile("^core id[ \t]*:[ \t]*([0-9]*)");

  /**
   * 解析/proc/stat的样板
   */
  private static final String PROCFS_STAT = "/proc/stat";
  private static final Pattern CPU_TIME_FORMAT =
      Pattern.compile("^cpu[ \t]*([0-9]*)" +
                      "[ \t]*([0-9]*)[ \t]*([0-9]*)[ \t].*");
  private CpuTimeTracker cpuTimeTracker;

  /**
   * 解析/proc/net/dev的样板
   */
  private static final String PROCFS_NETFILE = "/proc/net/dev";
  private static final Pattern PROCFS_NETFILE_FORMAT =
      Pattern .compile("^[ \t]*([a-zA-Z]+[0-9]*):" +
               "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" +
               "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" +
               "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" +
               "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+).*");

  /**
   * 解析/proc/diskstats的样板
   */
  private static final String PROCFS_DISKSFILE = "/proc/diskstats";
  private static final Pattern PROCFS_DISKSFILE_FORMAT =
      Pattern.compile("^[ \t]*([0-9]+)[ \t]*([0-9 ]+)" +
              "(?!([a-zA-Z]+[0-9]+))([a-zA-Z]+)" +
              "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" +
              "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)" +
              "[ \t]*([0-9]+)[ \t]*([0-9]+)[ \t]*([0-9]+)");
  /**
   * 解析/sys/block/partition_name/queue/hw_sector_size的样板
   */
  private static final Pattern PROCFS_DISKSECTORFILE_FORMAT =
      Pattern.compile("^([0-9]+)");

  private String procfsMemFile;
  private String procfsCpuFile;
  private String procfsStatFile;
  private String procfsNetFile;
  private String procfsDisksFile;
  private long jiffyLengthInMillis;

  private long ramSize = 0;
  private long swapSize = 0;
  private long ramSizeFree = 0;  // 机器上的可用 ram 磁盘空间 (kB)
  private long swapSizeFree = 0; // 机器上的可用 swap 空间 (kB)
  private long inactiveSize = 0; // 不活跃内存 (kB)
  private long inactiveFileSize = -1; // 非活动缓存,-1(如果不存在)
  private long hardwareCorruptSize = 0; // RAM已损坏且不可用大小
  private long hugePagesTotal = 0; // # 保留的标准大页
  private long hugePageSize = 0; // # 每个标准大页的大小

  /* 系统上的逻辑处理器数量. */
  private int numProcessors = 0;
  /* 系统上的物理核心数. */
  private int numCores = 0;
  private long cpuFrequency = 0L; // 系统上的CPU频率 (kHz)
  private long numNetBytesRead = 0L; // 从网络读取的聚合字节
  private long numNetBytesWritten = 0L; // 写入网络的聚合字节数
  private long numDisksBytesRead = 0L; // 从磁盘读取的聚合字节数
  private long numDisksBytesWritten = 0L; // 写入磁盘的聚合字节数

  private boolean readMemInfoFile = false;
  private boolean readCpuInfoFile = false;

  /* 为每个磁盘映射其扇区大小 */
  private HashMap<String, Integer> perDiskSectorSize = null;

  public static final long PAGE_SIZE = getConf("PAGESIZE");
  public static final long JIFFY_LENGTH_IN_MILLIS =
      Math.max(Math.round(1000D / getConf("CLK_TCK")), -1);

  private static long getConf(String attr) {
    if(Shell.LINUX) {
      try {
        ShellCommandExecutor shellExecutorClk = new ShellCommandExecutor(
            new String[] {"getconf", attr });
        shellExecutorClk.execute();
        return Long.parseLong(shellExecutorClk.getOutput().replace("\n", ""));
      } catch (IOException|NumberFormatException e) {
        return -1;
      }
    }
    return -1;
  }

  /**
   * 获取当前时间
   * @return Unix时间戳(毫秒)
   */
  long getCurrentTime() {
    return System.currentTimeMillis();
  }

  public SysInfoLinux() {
    this(PROCFS_MEMFILE, PROCFS_CPUINFO, PROCFS_STAT,
         PROCFS_NETFILE, PROCFS_DISKSFILE, JIFFY_LENGTH_IN_MILLIS);
  }

  /**
   * 构造函数,它允许分配/proc/目录。这将仅用于单元测试
   * @param procfsMemFile fake file for /proc/meminfo
   * @param procfsCpuFile fake file for /proc/cpuinfo
   * @param procfsStatFile fake file for /proc/stat
   * @param procfsNetFile fake file for /proc/net/dev
   * @param procfsDisksFile fake file for /proc/diskstats
   * @param jiffyLengthInMillis fake jiffy length value
   */
  @VisibleForTesting
  public SysInfoLinux(String procfsMemFile,
                                       String procfsCpuFile,
                                       String procfsStatFile,
                                       String procfsNetFile,
                                       String procfsDisksFile,
                                       long jiffyLengthInMillis) {
    this.procfsMemFile = procfsMemFile;
    this.procfsCpuFile = procfsCpuFile;
    this.procfsStatFile = procfsStatFile;
    this.procfsNetFile = procfsNetFile;
    this.procfsDisksFile = procfsDisksFile;
    this.jiffyLengthInMillis = jiffyLengthInMillis;
    this.cpuTimeTracker = new CpuTimeTracker(jiffyLengthInMillis);
    this.perDiskSectorSize = new HashMap<String, Integer>();
  }

  /**
   * 读 /proc/meminfo, 只解析和计算一次内存信息.
   */
  private void readProcMemInfoFile() {
    readProcMemInfoFile(false);
  }

  /**
   *
   * Long.parseLong()的包装器,如果值无效则返回零。在某些情况下,
   * /proc/meminfo中的swapFree可能为负数,报告为非常大的十进制值。
   */
  private long safeParseLong(String strVal) {
    long parsedVal;
    try {
      parsedVal = Long.parseLong(strVal);
    } catch (NumberFormatException nfe) {
      parsedVal = 0;
    }
    return parsedVal;
  }
  /**
   * 读 /proc/meminfo, 解析和计算内存信息.
   * @param readAgain if false, read only on the first time
   */
  private void readProcMemInfoFile(boolean readAgain) {

    if (readMemInfoFile && !readAgain) {
      return;
    }

    // Read "/proc/memInfo" file
    BufferedReader in;
    InputStreamReader fReader;
    try {
      fReader = new InputStreamReader(
          new FileInputStream(procfsMemFile), Charset.forName("UTF-8"));
      in = new BufferedReader(fReader);
    } catch (FileNotFoundException f) {
      // shouldn't happen....
      LOG.warn("Couldn't read " + procfsMemFile
          + "; can't determine memory settings");
      return;
    }

    Matcher mat;

    try {
      String str = in.readLine();
      while (str != null) {
        mat = PROCFS_MEMFILE_FORMAT.matcher(str);
        if (mat.find()) {
          if (mat.group(1).equals(MEMTOTAL_STRING)) {
            ramSize = Long.parseLong(mat.group(2));
          } else if (mat.group(1).equals(SWAPTOTAL_STRING)) {
            swapSize = Long.parseLong(mat.group(2));
          } else if (mat.group(1).equals(MEMFREE_STRING)) {
            ramSizeFree = safeParseLong(mat.group(2));
          } else if (mat.group(1).equals(SWAPFREE_STRING)) {
            swapSizeFree = safeParseLong(mat.group(2));
          } else if (mat.group(1).equals(INACTIVE_STRING)) {
            inactiveSize = Long.parseLong(mat.group(2));
          } else if (mat.group(1).equals(INACTIVEFILE_STRING)) {
            inactiveFileSize = Long.parseLong(mat.group(2));
          } else if (mat.group(1).equals(HARDWARECORRUPTED_STRING)) {
            hardwareCorruptSize = Long.parseLong(mat.group(2));
          } else if (mat.group(1).equals(HUGEPAGESTOTAL_STRING)) {
            hugePagesTotal = Long.parseLong(mat.group(2));
          } else if (mat.group(1).equals(HUGEPAGESIZE_STRING)) {
            hugePageSize = Long.parseLong(mat.group(2));
          }
        }
        str = in.readLine();
      }
    } catch (IOException io) {
      LOG.warn("Error reading the stream " + io);
    } finally {
      //关闭流
      try {
        fReader.close();
        try {
          in.close();
        } catch (IOException i) {
          LOG.warn("Error closing the stream " + in);
        }
      } catch (IOException i) {
        LOG.warn("Error closing the stream " + fReader);
      }
    }

    readMemInfoFile = true;
  }

  /**
   * 读 /proc/cpuinfo, 解析和计算CPU信息.
   */
  private void readProcCpuInfoFile() {
    // 此目录只需读取一次
    if (readCpuInfoFile) {
      return;
    }
    HashSet<String> coreIdSet = new HashSet<>();
    // Read "/proc/cpuinfo" file
    BufferedReader in;
    InputStreamReader fReader;
    try {
      fReader = new InputStreamReader(
          new FileInputStream(procfsCpuFile), Charset.forName("UTF-8"));
      in = new BufferedReader(fReader);
    } catch (FileNotFoundException f) {
      // shouldn't happen....
      LOG.warn("Couldn't read " + procfsCpuFile + "; can't determine cpu info");
      return;
    }
    Matcher mat;
    try {
      numProcessors = 0;
      numCores = 1;
      String currentPhysicalId = "";
      String str = in.readLine();
      while (str != null) {
        mat = PROCESSOR_FORMAT.matcher(str);
        if (mat.find()) {
          numProcessors++;
        }
        mat = FREQUENCY_FORMAT.matcher(str);
        if (mat.find()) {
          cpuFrequency = (long)(Double.parseDouble(mat.group(1)) * 1000); // kHz
        }
        mat = PHYSICAL_ID_FORMAT.matcher(str);
        if (mat.find()) {
          currentPhysicalId = str;
        }
        mat = CORE_ID_FORMAT.matcher(str);
        if (mat.find()) {
          coreIdSet.add(currentPhysicalId + " " + str);
          numCores = coreIdSet.size();
        }
        str = in.readLine();
      }
    } catch (IOException io) {
      LOG.warn("Error reading the stream " + io);
    } finally {
      // Close the streams
      try {
        fReader.close();
        try {
          in.close();
        } catch (IOException i) {
          LOG.warn("Error closing the stream " + in);
        }
      } catch (IOException i) {
        LOG.warn("Error closing the stream " + fReader);
      }
    }
    readCpuInfoFile = true;
  }

  /**
   * 读/proc/stat file, parse and calculate cumulative CPU.
   */
  private void readProcStatFile() {
    // Read "/proc/stat" file
    BufferedReader in;
    InputStreamReader fReader;
    try {
      fReader = new InputStreamReader(
          new FileInputStream(procfsStatFile), Charset.forName("UTF-8"));
      in = new BufferedReader(fReader);
    } catch (FileNotFoundException f) {
      // shouldn't happen....
      return;
    }

    Matcher mat;
    try {
      String str = in.readLine();
      while (str != null) {
        mat = CPU_TIME_FORMAT.matcher(str);
        if (mat.find()) {
          long uTime = Long.parseLong(mat.group(1));
          long nTime = Long.parseLong(mat.group(2));
          long sTime = Long.parseLong(mat.group(3));
          cpuTimeTracker.updateElapsedJiffies(
              BigInteger.valueOf(uTime + nTime + sTime),
              getCurrentTime());
          break;
        }
        str = in.readLine();
      }
    } catch (IOException io) {
      LOG.warn("Error reading the stream " + io);
    } finally {
      // Close the streams
      try {
        fReader.close();
        try {
          in.close();
        } catch (IOException i) {
          LOG.warn("Error closing the stream " + in);
        }
      } catch (IOException i) {
        LOG.warn("Error closing the stream " + fReader);
      }
    }
  }

  /**
   * Read /proc/net/dev file, 解析并计算通过网络读取和写入的字节数。
   */
  private void readProcNetInfoFile() {

    numNetBytesRead = 0L;
    numNetBytesWritten = 0L;

    // Read "/proc/net/dev" file
    BufferedReader in;
    InputStreamReader fReader;
    try {
      fReader = new InputStreamReader(
          new FileInputStream(procfsNetFile), Charset.forName("UTF-8"));
      in = new BufferedReader(fReader);
    } catch (FileNotFoundException f) {
      return;
    }

    Matcher mat;
    try {
      String str = in.readLine();
      while (str != null) {
        mat = PROCFS_NETFILE_FORMAT.matcher(str);
        if (mat.find()) {
          assert mat.groupCount() >= 16;

          // 忽略环回接口
          if (mat.group(1).equals("lo")) {
            str = in.readLine();
            continue;
          }
          numNetBytesRead += Long.parseLong(mat.group(2));
          numNetBytesWritten += Long.parseLong(mat.group(10));
        }
        str = in.readLine();
      }
    } catch (IOException io) {
      LOG.warn("Error reading the stream " + io);
    } finally {
      // Close the streams
      try {
        fReader.close();
        try {
          in.close();
        } catch (IOException i) {
          LOG.warn("Error closing the stream " + in);
        }
      } catch (IOException i) {
        LOG.warn("Error closing the stream " + fReader);
      }
    }
  }

  /**
   * Read /proc/diskstats file, 解析和计算从磁盘读取和写入磁盘的字节数。
   */
  private void readProcDisksInfoFile() {

    numDisksBytesRead = 0L;
    numDisksBytesWritten = 0L;

    // Read "/proc/diskstats" file
    BufferedReader in;
    try {
      in = new BufferedReader(new InputStreamReader(
            new FileInputStream(procfsDisksFile), Charset.forName("UTF-8")));
    } catch (FileNotFoundException f) {
      return;
    }

    Matcher mat;
    try {
      String str = in.readLine();
      while (str != null) {
        mat = PROCFS_DISKSFILE_FORMAT.matcher(str);
        if (mat.find()) {
          String diskName = mat.group(4);
          assert diskName != null;
          // 忽略循环或ram分区
          if (diskName.contains("loop") || diskName.contains("ram")) {
            str = in.readLine();
            continue;
          }

          Integer sectorSize;
          synchronized (perDiskSectorSize) {
            sectorSize = perDiskSectorSize.get(diskName);
            if (null == sectorSize) {
              // 检索扇区大小
              // 如果不可用或错误,假设512
              sectorSize = readDiskBlockInformation(diskName, 512);
              perDiskSectorSize.put(diskName, sectorSize);
            }
          }

          String sectorsRead = mat.group(7);
          String sectorsWritten = mat.group(11);
          if (null == sectorsRead || null == sectorsWritten) {
            return;
          }
          numDisksBytesRead += Long.parseLong(sectorsRead) * sectorSize;
          numDisksBytesWritten += Long.parseLong(sectorsWritten) * sectorSize;
        }
        str = in.readLine();
      }
    } catch (IOException e) {
      LOG.warn("Error reading the stream " + procfsDisksFile, e);
    } finally {
      // Close the streams
      try {
        in.close();
      } catch (IOException e) {
        LOG.warn("Error closing the stream " + procfsDisksFile, e);
      }
    }
  }

  /**
   * Read /sys/block/diskName/queue/hw_sector_size file, 解析并计算特定磁盘的扇区大小
   * @return sector size of specified disk, or defSector
   */
  int readDiskBlockInformation(String diskName, int defSector) {

    assert perDiskSectorSize != null && diskName != null;

    String procfsDiskSectorFile =
            "/sys/block/" + diskName + "/queue/hw_sector_size";

    BufferedReader in;
    try {
      in = new BufferedReader(new InputStreamReader(
            new FileInputStream(procfsDiskSectorFile),
              Charset.forName("UTF-8")));
    } catch (FileNotFoundException f) {
      return defSector;
    }

    Matcher mat;
    try {
      String str = in.readLine();
      while (str != null) {
        mat = PROCFS_DISKSECTORFILE_FORMAT.matcher(str);
        if (mat.find()) {
          String secSize = mat.group(1);
          if (secSize != null) {
            return Integer.parseInt(secSize);
          }
        }
        str = in.readLine();
      }
      return defSector;
    } catch (IOException|NumberFormatException e) {
      LOG.warn("Error reading the stream " + procfsDiskSectorFile, e);
      return defSector;
    } finally {
      // Close the streams
      try {
        in.close();
      } catch (IOException e) {
        LOG.warn("Error closing the stream " + procfsDiskSectorFile, e);
      }
    }
  }

  /** {@inheritDoc} */
  @Override
  public long getPhysicalMemorySize() {
    readProcMemInfoFile();
    return (ramSize
            - hardwareCorruptSize
            - (hugePagesTotal * hugePageSize)) * 1024;
  }

  /** {@inheritDoc} */
  @Override
  public long getVirtualMemorySize() {
    return getPhysicalMemorySize() + (swapSize * 1024);
  }

  /** {@inheritDoc} */
  @Override
  public long getAvailablePhysicalMemorySize() {
    readProcMemInfoFile(true);
    long inactive = inactiveFileSize != -1
        ? inactiveFileSize
        : inactiveSize;
    return (ramSizeFree + inactive) * 1024;
  }

  /** {@inheritDoc} */
  @Override
  public long getAvailableVirtualMemorySize() {
    return getAvailablePhysicalMemorySize() + (swapSizeFree * 1024);
  }

  /** {@inheritDoc} */
  @Override
  public int getNumProcessors() {
    readProcCpuInfoFile();
    return numProcessors;
  }

  /** {@inheritDoc} */
  @Override
  public int getNumCores() {
    readProcCpuInfoFile();
    return numCores;
  }

  /** {@inheritDoc} */
  @Override
  public long getCpuFrequency() {
    readProcCpuInfoFile();
    return cpuFrequency;
  }

  /** {@inheritDoc} */
  @Override
  public long getCumulativeCpuTime() {
    readProcStatFile();
    return cpuTimeTracker.getCumulativeCpuTime();
  }

  /** {@inheritDoc} */
  @Override
  public float getCpuUsagePercentage() {
    readProcStatFile();
    float overallCpuUsage = cpuTimeTracker.getCpuTrackerUsagePercent();
    if (overallCpuUsage != CpuTimeTracker.UNAVAILABLE) {
      overallCpuUsage = overallCpuUsage / getNumProcessors();
    }
    return overallCpuUsage;
  }

  /** {@inheritDoc} */
  @Override
  public float getNumVCoresUsed() {
    readProcStatFile();
    float overallVCoresUsage = cpuTimeTracker.getCpuTrackerUsagePercent();
    if (overallVCoresUsage != CpuTimeTracker.UNAVAILABLE) {
      overallVCoresUsage = overallVCoresUsage / 100F;
    }
    return overallVCoresUsage;
  }

  /** {@inheritDoc} */
  @Override
  public long getNetworkBytesRead() {
    readProcNetInfoFile();
    return numNetBytesRead;
  }

  /** {@inheritDoc} */
  @Override
  public long getNetworkBytesWritten() {
    readProcNetInfoFile();
    return numNetBytesWritten;
  }

  @Override
  public long getStorageBytesRead() {
    readProcDisksInfoFile();
    return numDisksBytesRead;
  }

  @Override
  public long getStorageBytesWritten() {
    readProcDisksInfoFile();
    return numDisksBytesWritten;
  }

  /**
   * 测试 {@link SysInfoLinux}.
   *
   * @param args - arguments to this calculator test
   */
  public static void main(String[] args) {
    SysInfoLinux plugin = new SysInfoLinux();
    System.out.println("Physical memory Size (bytes) : "
        + plugin.getPhysicalMemorySize());
    System.out.println("Total Virtual memory Size (bytes) : "
        + plugin.getVirtualMemorySize());
    System.out.println("Available Physical memory Size (bytes) : "
        + plugin.getAvailablePhysicalMemorySize());
    System.out.println("Total Available Virtual memory Size (bytes) : "
        + plugin.getAvailableVirtualMemorySize());
    System.out.println("Number of Processors : " + plugin.getNumProcessors());
    System.out.println("CPU frequency (kHz) : " + plugin.getCpuFrequency());
    System.out.println("Cumulative CPU time (ms) : " +
            plugin.getCumulativeCpuTime());
    System.out.println("Total network read (bytes) : "
            + plugin.getNetworkBytesRead());
    System.out.println("Total network written (bytes) : "
            + plugin.getNetworkBytesWritten());
    System.out.println("Total storage read (bytes) : "
            + plugin.getStorageBytesRead());
    System.out.println("Total storage written (bytes) : "
            + plugin.getStorageBytesWritten());
    try {
      // 睡眠,以便计算CPU使用量
      Thread.sleep(500L);
    } catch (InterruptedException e) {
      // do nothing
    }
    System.out.println("CPU usage % : " + plugin.getCpuUsagePercentage());
  }

  @VisibleForTesting
  void setReadCpuInfoFile(boolean readCpuInfoFileValue) {
    this.readCpuInfoFile = readCpuInfoFileValue;
  }

  public long getJiffyLengthInMillis() {
    return this.jiffyLengthInMillis;
  }
}

从源码中可以看到,linux上的资源信息是从各个文件中获取的:

/proc/meminfo 解析和计算内存信息
/proc/cpuinfo 解析和计算CPU信息
/proc/stat 获取CPU的运行数据
/proc/net/dev 解析并计算通过网络读取和写入的字节数。
/proc/diskstats 解析和计算从磁盘读取和写入磁盘的字节数
/sys/block/partition_name/queue/hw_sector_size 解析并计算特定磁盘的扇区大小

下面我们看下linux操作系统中的这些文件中都是什么内容

三、proc下各示例文件

1、/proc/meminfo

MemTotal: 131479404 kB
MemFree: 29109144 kB
MemAvailable: 109564132 kB
Buffers: 3922940 kB
Cached: 65545372 kB
SwapCached: 0 kB
Active: 9252304 kB
Inactive: 79107168 kB
Active(anon): 120256 kB
Inactive(anon): 18893260 kB
Active(file): 9132048 kB
Inactive(file): 60213908 kB
Unevictable: 0 kB
Mlocked: 0 kB
SwapTotal: 0 kB
SwapFree: 0 kB
Dirty: 123956 kB
Writeback: 0 kB
AnonPages: 18856940 kB
Mapped: 151268 kB
Shmem: 122260 kB
KReclaimable: 12166144 kB
Slab: 13454424 kB
SReclaimable: 12166144 kB
SUnreclaim: 1288280 kB
KernelStack: 29584 kB
PageTables: 88028 kB
NFS_Unstable: 0 kB
Bounce: 0 kB
WritebackTmp: 0 kB
CommitLimit: 65739700 kB
Committed_AS: 34508552 kB
VmallocTotal: 34359738367 kB
VmallocUsed: 168540 kB
VmallocChunk: 0 kB
Percpu: 210432 kB
HardwareCorrupted: 0 kB
AnonHugePages: 12728320 kB
ShmemHugePages: 0 kB
ShmemPmdMapped: 0 kB
FileHugePages: 0 kB
FilePmdMapped: 0 kB
DupText: 0 kB
HugePages_Total: 0
HugePages_Free: 0
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize: 2048 kB
Hugetlb: 0 kB
DirectMap4k: 2277228 kB
DirectMap2M: 119357440 kB
DirectMap1G: 14680064 kB

/proc/meminfo是了解Linux系统内存使用状况的主要接口,我们最常用的”free”、”vmstat”等命令就是通过它获取数据的

MemTotal:系统从加电开始到引导完成,firmware/BIOS要保留一些内存,kernel本身要占用一些内存,最后剩下可供kernel支配的内存就是MemTotal。这个值在系统运行期间一般是固定不变的

MemFree:表示系统尚未使用的内存。[MemTotal-MemFree]就是已被用掉的内存

MemAvailable:有些应用程序会根据系统的可用内存大小自动调整内存申请的多少,所以需要一个记录当前可用内存数量的统计值,MemFree并不适用,因为MemFree不能代表全部可用的内存,系统中有些内存虽然已被使用但是可以回收的,比如cache/buffer、slab都有一部分可以回收,所以这部分可回收的内存加上MemFree才是系统可用的内存,即MemAvailable。/proc/meminfo中的MemAvailable是内核使用特定的算法估算出来的,要注意这是一个估计值,并不精确

Buffers:用于缓存数据的内存量

Cached:用于缓存文件系统数据的内存量

SwapCached:用于缓存交换分区数据的内存量

Active:活跃内存量,活跃指最近被访问过或正在使用中的内存

Inactive:非活跃内存量,非活跃指长时间没有被访问或已经释放但仍在缓冲区中的内存

Active(anon):活跃的匿名内存

Inactive(anon):不活跃的匿名内存

Active(file):活跃的文件使用内存

Inactive(file):不活跃的文件使用内存

Unevictable:不能被释放的内存页

Mlocked:系统调用 mlock 家族允许程序在物理内存上锁住它的部分或全部地址空间。这将阻止Linux 将这个内存页调度到交换空间(swap space),即使该程序已有一段时间没有访问这段空间

SwapTotal:交换分区总容量

SwapFree:交换分区剩余容量

Dirty:脏页面数量,脏页面是指已经被修改但还没有被写回磁盘的页面

Writeback:等待写回磁盘的页面数量

AnonPages:未映射页的内存大小

Mapped:设备和文件等映射的大小

Shmem:包含共享内存以及

tmpfs

文件系统占用的内存

KReclaimable:

SReclaimable

的基础上加上

MISC_RECLAIMABLE

Slab:Slab是Linux中用来管理小块内核对象分配和释放的一种内存管理机制,Slab = SReclaimable + SUnreclaim

SReclaimable:slab中可回收的部分

SUnreclaim:slab中不可回收的部分

KernelStack:内核消耗的内存

PageTables:管理内存分页的索引表的大小

NFS_Unstable:不稳定页表的大小

Bounce:在低端内存中分配一个临时buffer作为跳转,把位于高端内存的缓存数据复制到此处消耗的内存

WritebackTmp:FUSE用于临时写回缓冲区的内存

CommitLimit:系统实际可分配内存

Committed_AS:系统当前已分配的内存

VmallocTotal:预留的虚拟内存总量

VmallocUsed:已经被使用的虚拟内存

VmallocChunk:可分配的最大的逻辑连续的虚拟内存

Percpu:表示用于percpu分配的的内存大小,不包括metadata

HardwareCorrupted:当系统检测到内存的硬件故障时删除掉的内存页的总量

AnonHugePages:匿名大页缓存

ShmemHugePages:表示用于shared memory或tmpfs的透明大页

ShmemPmdMapped:表示用于用户态shared memory映射的透明大页

FileHugePages:

FilePmdMapped:

DupText:

HugePages_Total:预留的大页内存总量

HugePages_Free:空闲的大页内存

HugePages_Rsvd:已经被应用程序分配但尚未使用的大页内存

HugePages_Surp:初始大页数与修改配置后大页数的差值

Hugepagesize:单个大页内存的大小

Hugetlb:

DirectMap4k:映射TLB为4kB的内存数量

DirectMap2M:映射TLB为2M的内存数量

DirectMap1G:映射TLB为1G的内存数量

2、/proc/cpuinfo

processor : 0
vendor_id : GenuineIntel
cpu family : 6
model : 85
model name : Intel Xeon Processor (Skylake, IBRS)
stepping : 4
microcode : 0x1
cpu MHz : 2299.992
cache size : 16384 KB
physical id : 0
siblings : 12
core id : 0
cpu cores : 12
apicid : 0
initial apicid : 0
fpu : yes
fpu_exception : yes
cpuid level : 13
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology c
puid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault invpcid
_single ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xge
tbv1 xsaves arat umip pku ospke md_clear
bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs taa itlb_multihit mmio_stale_data retbleed
bogomips : 4599.98
clflush size : 64
cache_alignment : 64
address sizes : 46 bits physical, 48 bits virtual
power management:

.................................省略...............................

processor : 23
vendor_id : GenuineIntel
cpu family : 6
model : 85
model name : Intel Xeon Processor (Skylake, IBRS)
stepping : 4
microcode : 0x1
cpu MHz : 2299.992
cache size : 16384 KB
physical id : 1
siblings : 12
core id : 11
cpu cores : 12
apicid : 27
initial apicid : 27
fpu : yes
fpu_exception : yes
cpuid level : 13
wp : yes
flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault invpcid_single ssbd ibrs ibpb stibp fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves arat umip pku ospke md_clear
bugs : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs taa itlb_multihit mmio_stale_data retbleed
bogomips : 4599.98
clflush size : 64
cache_alignment : 64
address sizes : 46 bits physical, 48 bits virtual
power management:

通过/proc/cpuinfo,可以查看系统中CPU的提供商和相关配置信息,例如我们想获取,有多少颗物理CPU,每个物理cpu核心数,以及超线程是否开启等信息,下面是每项参数的解释:

**processor **:系统中逻辑处理核心数的编号,从0开始排序

**vendor_id **:CPU制造商

cpu family :CPU产品系列代号

**model **:CPU属于其系列中的哪一代的代号

model name:CPU属于的名字及其编号、标称主频

stepping :CPU属于制作更新版本

cpu MHz :CPU的实际使用主频

cache size :CPU二级缓存大小

physical id :单个物理CPU的标号

siblings :单个物理CPU的逻辑CPU数。siblings=cpu cores [*2]

core id :当前物理核在其所处CPU中的编号,这个编号不一定连续

cpu cores :该逻辑核所处CPU的物理核数。比如此处cpu cores 是4个,那么对应core id 可能是 1、3、4、5。

apicid :用来区分不同逻辑核的编号,系统中每个逻辑核的此编号必然不同,此编号不一定连续

fpu :是否具有浮点运算单元(Floating Point Unit)

fpu_exception :是否支持浮点计算异常

cpuid level :执行cpuid指令前,eax寄存器中的值,根据不同的值cpuid指令会返回不同的内容

wp :表明当前CPU是否在内核态支持对用户空间的写保护(Write Protection)

flags :当前CPU支持的功能

bugs :保存内核检测到的硬件错误

bogomips:在系统内核启动时粗略测算的CPU速度(Million Instructions Per Second

clflush size :每次刷新缓存的大小单位

cache_alignment :缓存地址对齐单位

address sizes :可访问地址空间位数

**power management **:对能源管理的支持

3、/proc/stat

cpu 8736131408 5678 193273561 20516569092 64420147 202089758 24839900 736849109 0 0
cpu0 248135539 221 7727953 977102347 3547852 7843342 1836321 26959080 0 0
cpu1 288960610 208 7253118 939752913 3465875 6253758 983012 22610655 0 0
cpu2 331782665 173 6580943 898614791 3179773 5927649 705463 20377701 0 0
cpu3 357312958 119 6079401 873515056 3082820 5636253 675032 19192651 0 0
cpu4 416415379 134 4970987 817517624 2702326 5029001 461878 15707695 0 0
cpu5 431656991 49 4560783 803250964 2571613 4865074 410522 14757252 0 0
cpu6 435254195 169 4473639 799793313 2533305 4816086 391800 14568012 0 0
cpu7 430149966 193 5306474 799970897 2274711 5119291 4203530 15644401 0 0
cpu8 442943177 38 4245770 792483084 2429873 4720225 472371 14247201 0 0
cpu9 450045462 66 4306296 784947171 2410185 4660788 1014418 13736325 0 0
cpu10 451355787 39 4064980 784676208 2261500 4619102 396365 13709422 0 0
cpu11 457452811 64 3901559 778822656 2231050 4565301 393815 13362096 0 0
cpu12 308242715 360 10256742 887773491 2996516 16295533 1139546 52646450 0 0
cpu13 291977604 326 12356349 906520116 2948164 14127718 995623 49241553 0 0
cpu14 314799511 763 11789179 887439430 2724265 12548004 926359 45887607 0 0
cpu15 343893959 589 10565703 861975365 2476760 11234089 879714 42636791 0 0
cpu16 357686825 202 9889055 850148237 2299417 10596337 839523 40920853 0 0
cpu17 371250803 336 9230196 838222800 2170862 10053101 814107 39442271 0 0
cpu18 381559924 227 8684083 829076848 2133621 9640147 793837 38300080 0 0
cpu19 391204359 360 8217143 820527206 2044874 9294489 780223 37347635 0 0
cpu20 395787749 376 7934865 816504952 2030798 9101060 793620 36768582 0 0
cpu21 382765684 287 8586382 826536768 2751789 9517617 1313220 39011440 0 0
cpu22 403637541 284 7699240 808560612 3025058 8623423 868515 35789081 0 0
cpu23 51859181 83 24592711 1132836231 4127127 17002359 2751076 73984264 0 0
intr 41293626138 0 9 0 0 250 0 0 0 0 0 1257757 0 15 0 12281947 0 0 0 0 0 0 0 0 0 0 74418226 0 9915152 0 62205235 0 16892400 0 170973248 4915 0 3076909346 41960 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
ctxt 66125412027
btime 1695643289
processes 68123558
procs_running 20
procs_blocked 0
softirq 26896249797 0 1534546890 2296517 681491783 147607814 0 4570937 2751393260 9053148 290452968

/proc/stat记录的是系统进程整体的统计信息

cpu ~ cpu23 描述的指标如下:(单位都是jiffies,1jiffies=0.01秒)
nameusernicesystemidleiowaitirrqsoftirqstealguestguest_nicecpu8736131408567819327356120516569092644201472020897582483990073684910900…………………………………………………………cpu23518591818324592711113283623141271271700235927510767398426400
这部分统计了CPU时间消耗分布在哪些地方,是用户态任务,还是内核或者是中断、idle等等;第一行的第一个字段为"cpu",没有指定CPU号,表示是其他各个CPU时间分布的汇总;而其他行有cpu编号表示特定编号的cpu时间消耗分布情况。

name:cpu汇总/cpu编号

user:到目前为止,CPU上nice值不大于0的用户态任务的运行时间

nice:到目前为止,CPU上nice值大于0的用户态任务的运行时间

system:到目前为止,CPU上内核态的运行时间。包括用户态任务系统调用、异常等陷入内核消耗时间,也包括内核线程消耗的时间; 但是不包括中断和软中断的执行时间

idle:到目前为止,处于idle任务的时间。不包括CPU上因为任务IO阻塞导致CPU上没有任务可运行、处于idle状态的时间。

iowait:到目前为止,由于CPU上任务IO阻塞导致CPU无可运行任务、处于idle的时间。需要强调的是,iowait是在CPU处于idle状态下的一种特殊情况的时间,与上面的“idle”列互补构成CPU上真正处于idle的时间。

irrq:到目前为止,CPU进入到中断处理的时间(在没有使能CONFIG_IRQ_TIME_ACCOUNTING的情况下,大部分架构实际上无法记录到该项,见下面的分析)。

softirq:到目前为止,CPU处理软中断的时间,包括softirqd中处理软中断的时间。

steal:与虚拟化有关

guest:与虚拟化有关

guest_nice:与虚拟化有关

其他指标如下:

intr:系统启动以来的所有interrupts的次数情况

ctxt: 系统上下文切换次数

btime:启动时长(单位:秒),从Epoch(即1970零时)开始到系统启动所经过的时长,每次启动会改变。此处指为1695643289,转换北京时间为2023-09-25 20:01:29

processes:系统启动后所创建过的进程数量。当短时间该值特别大,系统可能出现异常

procs_running:处于Runnable状态的进程个数

procs_blocked:处于被阻塞的进程个数

softirq:此行显示所有CPU的softirq总数,第一列是所有软件和每个软件的总数,后面的列是特定softirq的总数

4、/proc/net/dev

Inter-| Receive | Transmit
face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed
lo: 8262570189990 1009625458 0 1217 0 0 0 0 8262570189990 1009625458 0 0 0 0 0 0
eth0: 4725352845467 3727035213 0 0 0 0 0 0 12125473603784 322028286 0 0 0 0 0 0
eth1: 12250497813726 9789277543 0 1066 0 0 0 0 12066904463831 2712923220 0 0 0 0 0 0

/proc/net/dev提供了网络设备接口的统计信息,lo、eth0、eth1为网络社保接口的名称。下面是具体指标解释

bytes:接收或发送的总字节数。

packets:接收或发送的总数据包数量。

errs:在接收或发送过程中发生的错误数量。

drop:由于缓冲区溢出或其他原因而丢弃的数据包数量。

fifo:由于队列拥塞而丢弃的数据包数量。

frame:由于帧错误而丢弃的数据包数量。

compressed:压缩传输所节省的字节数量。

multicast:接收到多播数据包的数量。

5、/proc/diskstats

253 0 vda 978184 4128 81150594 20755299 73453182 4078455 1309816689 398292145 0 113223744 426791458 0 0 0 0 15585634 7744013 20200452 365556940 0
253 1 vda1 977999 4128 81131978 20753990 72102739 4078455 1309816681 396863176 0 112567056 417617166 0 0 0 0 0 0 20199145 365556938 0
253 16 vdb 10371383 566918 5197295816 186317736 4253043 5537854 2637106896 128050659 0 68754698 314653319 0 0 0 0 411815 284922 185895864 126318932 0
253 17 vdb1 10371274 566918 5197280288 186316863 4253012 5537854 2637106896 128050617 0 68754916 314367481 0 0 0 0 0 0 185894994 126318932 0
253 32 vdc 62355261 2902021 51036086962 4222776827 51209471 48829571 38641223240 1144000034 1 453445155 1073094954 0 0 0 0 1186448 1285388 4096018303 1082051035 0
11 0 sr0 352 0 3115 420 0 0 0 0 0 456 420 0 0 0 0 0 0 419 0 0

/proc/diskstatsv提供了磁盘信息的统计信息,含义从左到右依次如下

设备号 编号 设备 读完成次数 合并完成次数 读扇区次数 读操作花费毫秒数 写完成次数 合并写完成次数 写扇区次数 写操作花费的毫秒数 正在处理的输入/输出请求数 输入/输出操作花费的毫秒数 输入/输出操作花费的加权毫秒数

6、/sys/block/partition_name/queue/hw_sector_size

通过hw_sector_size可以获取磁盘块大小相关的信息。我们把partition_name换成vda、vdb、vdc或者sr0即可,注意:下面的512就是文件的内容

vda、vdb、vdc的磁盘块大小为

512

sr0的磁盘块大小为

2048

标签: hadoop 大数据 linux

本文转载自: https://blog.csdn.net/lu070828/article/details/136146608
版权归原作者 隔着天花板看星星 所有, 如有侵权,请联系我们删除。

“Hadoop-Yarn-NodeManager如何计算Linux系统上的资源信息”的评论:

还没有评论