#! /usr/bin/env python3 import re import sys import numpy as np import pandas as pd def parse_start_date(line): # Get start date from line in ISO 8601 format hdr = re.compile(r'''.*?\)\s+(?P\d+\-\d+\-\d+).*''') match_hdr = hdr.match(line) start_date = None if match_hdr: start_date = match_hdr['start'] return start_date class ParseInterface(object): def __init__(self, start_date): self.regex_hdr = None self.regex_data = None self.regex_footer = re.compile(r'''Average:.*''') self.start = start_date self.last_date = None self.parquet_name = "sar.parquet" self.fields = [] # Pass in a dict s: # { date: string in YYYY-MM-DD # time: string in hh:mm:ss # } # Pass in a last_date as an np.datetime64 obj or None # Returns a np.datetime64 object def parse_time(self, s, last_date): d = np.datetime64("{} {}".format(s['date'], s['time'])) if last_date: while (d - last_date) < np.timedelta64(0, 's'): d = d + np.timedelta64(1, 'D') return d def parse_data(self, f, save_parquet=True): line = f.readline() data = {} for key in self.fields: data[key[0]] = [] while(line): match_data = self.regex_data.match(line) if match_data: s = {'date': self.start, 'time': match_data['time']} d = self.parse_time(s, self.last_date) data['time'].append(d) self.last_date = d # Every other field is not special for key in self.fields[1:]: data[key[0]].append(key[1](match_data[key[0]])) line = f.readline() continue match_footer = self.regex_footer.match(line) if match_footer: break line = f.readline() df = pd.DataFrame(data) df = df.set_index('time') if (save_parquet): df.to_parquet(self.parquet_name, compression='gzip') return df # Look for the header, if we find it, read until we hit the end of the section # Return the data frame if we get one. def parse_for_header(self, line, f, save_parquet=True): match = self.regex_hdr.match(line) if match: return self.parse_data(f, save_parquet) return None class ParseIfaceUtil(ParseInterface): def __init__(self, start_date, parquet=None): super().__init__(start_date) self.regex_hdr = re.compile(r'''(?P\d+:\d+:\d+)\s+IFACE\s+''' r'''rxpck/s\s+txpck/s\s+rxkB/s\s+txkB/s\s+rxcmp/s\s+txcmp/s\s+''' r'''rxmcst/s''') self.regex_data = re.compile(r'''(?P\d+:\d+:\d+)\s+(?P[\d\w]+)\s+''' r'''(?P\d+\.\d+)\s+(?P\d+\.\d+)\s+(?P\d+\.\d+)\s+''' r'''(?P\d+\.\d+)\s+(?P\d+\.\d+)\s+(?P\d+\.\d+)\s+''' r'''(?P\d+\.\d+)''') self.fields = [('time', None), ('iface', str), ('rxpcks', float), ('txpcks', float), ('rxkBs', float), ('txkBs', float), ('rxcmps', float), ('txcmps', float), ('rxmcsts', float)] self.start = start_date self.last_date = None if parquet: self.parquet_name = "sar_iface_{}.parquet".format(parquet) else: self.parquet_name = "sar_iface.parquet" # class that embodies the state machine for parsing SAR log for device utilization class ParseDevUtil(ParseInterface): def __init__(self, start_date, parquet=None): super().__init__(start_date) self.regex_hdr = re.compile(r'''(?P\d+:\d+:\d+)\s+DEV\s+tps\s+''' r'''rd_sec/s\s+wr_sec/s\s+avgrq\-sz\s+avgqu\-sz\s+''' r'''await\s+svctm\s+%util''') self.regex_data = re.compile(r'''(?P\d+:\d+:\d+)\s+(?P[\w\d\-]+)\s+(?P\d+\.\d+)\s+''' r'''(?P\d+\.\d+)\s+(?P\d+\.\d+)\s+(?P\d+\.\d+)\s+''' r'''(?P\d+\.\d+)\s+(?P\d+\.\d+)\s+(?P\d+\.\d+)\s+(?P\d+\.\d+)''') self.fields = [('time', None), ('dev', str), ('tps', float), ('rdsecs', float), ('wrsecs', float), ('avgrqsz', float), ('avgqusz', float), ('await', float), ('svctm', float), ('util', float)] self.start = start_date self.last_date = None if parquet: self.parquet_name = "sar_dev_{}.parquet".format(parquet) else: self.parquet_name = "sar_dev.parquet" # class that embodies the state machine for parsing SAR log for disk reads/writes class ParseDiskUtil(ParseInterface): def __init__(self, start_date, parquet=None): super().__init__(start_date) self.regex_hdr = re.compile(r'''(?P\d+:\d+:\d+)\s+tps\s+rtps\s+''' r'''wtps\s+bread/s\s+bwrtn/s''') self.regex_data = re.compile(r'''(?P\d+:\d+:\d+)\s+(?P\d+\.\d+)\s+''' r'''(?P\d+\.\d+)\s+(?P\d+\.\d+)\s+''' r'''(?P\d+\.\d+)\s+(?P\d+\.\d+)''') self.fields = [('time', None), ('tps', float), ('rtps', float), ('wtps', float), ('breads', float), ('bwrtns', float)] self.start = start_date self.last_date = None if parquet: self.parquet_name = "sar_disk_{}.parquet".format(parquet) else: self.parquet_name = "sar_disk.parquet" # class that embodies the state machine for parsing SAR logs for cpu utilization class ParseTcpTime(ParseInterface): def __init__(self, start_date, parquet=None): super().__init__(start_date) self.regex_hdr = re.compile(r'''(?P\d+:\d+:\d+)\s+active/s\s+''' r'''passive/s\s+iseg/s\s+oseg/s''') self.regex_data = re.compile(r'''(?P\d+:\d+:\d+)\s+(?P\d+\.\d+)\s+''' r'''(?P\d+\.\d+)\s+(?P\d+\.\d+)\s+''' r'''(?P\d+\.\d+)''') self.fields = [('time', None), ('active', str), ('passive', float), ('iseg', float), ('oseg', float)] self.start = start_date self.last_date = None if parquet: self.parquet_name = "sar_tcp_{}.parquet".format(parquet) else: self.parquet_name = "sar_tcp.parquet" # class that embodies the state machine for parsing SAR logs for cpu utilization class ParseCpuTime(ParseInterface): def __init__(self, start_date, parquet=None): super().__init__(start_date) self.regex_hdr = re.compile(r'''(?P\d+:\d+:\d+)\s+CPU\s+''' r'''%usr\s+%nice\s+%sys\s+%iowait\s+%steal\s+%irq\s+%soft\s+%guest\s+%gnice\s+%idle''') self.regex_data = re.compile(r'''(?P\d+:\d+:\d+)\s+(?P[\d\w]+)\s+''' r'''(?P\d+\.\d+)\s+(?P\d+\.\d+)\s+(?P\d+''' r'''\.\d+)\s+(?P\d+\.\d+)\s+(?P\d+\.\d+)''' r'''\s+(?P\d+\.\d+)\s+(?P\d+\.\d+)\s+''' r'''(?P\d+\.\d+)\s+(?P\d+\.\d+)\s+(?P\d+\.\d+)''') self.fields = [('time', None), ('cpu', str), ('usr', float), ('nice', float), ('sys', float), ('iowait', float), ('steal', float), ('irq', float), ('soft', float), ('guest', float), ('gnice', float), ('idle', float)] self.start = start_date self.last_date = None if parquet: self.parquet_name = "sar_cpu_{}.parquet".format(parquet) else: self.parquet_name = "sar_cpu.parquet" class ParseCSwitchTime(ParseInterface): def __init__(self, start_date, parquet=None): super().__init__(start_date) self.regex_hdr = re.compile(r'''(?P\d+:\d+:\d+)\s+proc/s\s+cswch/s''') self.regex_data = re.compile(r'''(?P\d+:\d+:\d+)\s+(?P\d+\.\d+)\s+(?P\d+\.\d+)''') self.start = start_date self.fields = [('time', None), ('proc_s', float), ('cswch_s', float)] self.last_date = None if parquet: self.parquet_name = "sar_cswch_{}.parquet".format(parquet) else: self.parquet_name = "sar_cswch.parquet" def parse_sysstat(file_name, suffix=None): with open(file_name, 'r') as f: # Get start date line = f.readline() start_date = parse_start_date(line) if not start_date: print("ERR: header not first line of Sar file, exiting") return 1 # Initialize parsers parseCPU = ParseCpuTime(start_date, parquet=suffix) parseDisk = ParseDiskUtil(start_date, parquet=suffix) parseDev = ParseDevUtil(start_date, parquet=suffix) parseIface = ParseIfaceUtil(start_date, parquet=suffix) parseTcpTime = ParseTcpTime(start_date, parquet=suffix) parseCswitch = ParseCSwitchTime(start_date, parquet=suffix) line = f.readline() while (line): parseCPU.parse_for_header(line, f) parseDisk.parse_for_header(line, f) parseDev.parse_for_header(line, f) parseIface.parse_for_header(line, f) parseTcpTime.parse_for_header(line, f) parseCswitch.parse_for_header(line, f) line = f.readline() return 0 if __name__ == "__main__": exit(parse_sysstat(sys.argv[1]))