perf/tests/attr.py

0001 # SPDX-License-Identifier: GPL-2.0
0002
0003 from __future__ import print_function
0004
0005 import os
0006 import sys
0007 import glob
0008 import optparse
0009 import tempfile
0010 import logging
0011 import shutil
0012
0013 try:
0014     import configparser
0015 except ImportError:
0016     import ConfigParser as configparser
0017
0018 def data_equal(a, b):
0019     # Allow multiple values in assignment separated by '|'
0020     a_list = a.split('|')
0021     b_list = b.split('|')
0022
0023     for a_item in a_list:
0024         for b_item in b_list:
0025             if (a_item == b_item):
0026                 return True
0027             elif (a_item == '*') or (b_item == '*'):
0028                 return True
0029
0030     return False
0031
0032 class Fail(Exception):
0033     def __init__(self, test, msg):
0034         self.msg = msg
0035         self.test = test
0036     def getMsg(self):
0037         return '\'%s\' - %s' % (self.test.path, self.msg)
0038
0039 class Notest(Exception):
0040     def __init__(self, test, arch):
0041         self.arch = arch
0042         self.test = test
0043     def getMsg(self):
0044         return '[%s] \'%s\'' % (self.arch, self.test.path)
0045
0046 class Unsup(Exception):
0047     def __init__(self, test):
0048         self.test = test
0049     def getMsg(self):
0050         return '\'%s\'' % self.test.path
0051
0052 class Event(dict):
0053     terms = [
0054         'cpu',
0055         'flags',
0056         'type',
0057         'size',
0058         'config',
0059         'sample_period',
0060         'sample_type',
0061         'read_format',
0062         'disabled',
0063         'inherit',
0064         'pinned',
0065         'exclusive',
0066         'exclude_user',
0067         'exclude_kernel',
0068         'exclude_hv',
0069         'exclude_idle',
0070         'mmap',
0071         'comm',
0072         'freq',
0073         'inherit_stat',
0074         'enable_on_exec',
0075         'task',
0076         'watermark',
0077         'precise_ip',
0078         'mmap_data',
0079         'sample_id_all',
0080         'exclude_host',
0081         'exclude_guest',
0082         'exclude_callchain_kernel',
0083         'exclude_callchain_user',
0084         'wakeup_events',
0085         'bp_type',
0086         'config1',
0087         'config2',
0088         'branch_sample_type',
0089         'sample_regs_user',
0090         'sample_stack_user',
0091     ]
0092
0093     def add(self, data):
0094         for key, val in data:
0095             log.debug("      %s = %s" % (key, val))
0096             self[key] = val
0097
0098     def __init__(self, name, data, base):
0099         log.debug("    Event %s" % name);
0100         self.name  = name;
0101         self.group = ''
0102         self.add(base)
0103         self.add(data)
0104
0105     def equal(self, other):
0106         for t in Event.terms:
0107             log.debug("      [%s] %s %s" % (t, self[t], other[t]));
0108             if t not in self or t not in other:
0109                 return False
0110             if not data_equal(self[t], other[t]):
0111                 return False
0112         return True
0113
0114     def optional(self):
0115         if 'optional' in self and self['optional'] == '1':
0116             return True
0117         return False
0118
0119     def diff(self, other):
0120         for t in Event.terms:
0121             if t not in self or t not in other:
0122                 continue
0123             if not data_equal(self[t], other[t]):
0124                 log.warning("expected %s=%s, got %s" % (t, self[t], other[t]))
0125
0126 # Test file description needs to have following sections:
0127 # [config]
0128 #   - just single instance in file
0129 #   - needs to specify:
0130 #     'command' - perf command name
0131 #     'args'    - special command arguments
0132 #     'ret'     - expected command return value (0 by default)
0133 #     'arch'    - architecture specific test (optional)
0134 #                 comma separated list, ! at the beginning
0135 #                 negates it.
0136 #
0137 # [eventX:base]
0138 #   - one or multiple instances in file
0139 #   - expected values assignments
0140 class Test(object):
0141     def __init__(self, path, options):
0142         parser = configparser.SafeConfigParser()
0143         parser.read(path)
0144
0145         log.warning("running '%s'" % path)
0146
0147         self.path     = path
0148         self.test_dir = options.test_dir
0149         self.perf     = options.perf
0150         self.command  = parser.get('config', 'command')
0151         self.args     = parser.get('config', 'args')
0152
0153         try:
0154             self.ret  = parser.get('config', 'ret')
0155         except:
0156             self.ret  = 0
0157
0158         try:
0159             self.arch  = parser.get('config', 'arch')
0160             log.warning("test limitation '%s'" % self.arch)
0161         except:
0162             self.arch  = ''
0163
0164         self.expect   = {}
0165         self.result   = {}
0166         log.debug("  loading expected events");
0167         self.load_events(path, self.expect)
0168
0169     def is_event(self, name):
0170         if name.find("event") == -1:
0171             return False
0172         else:
0173             return True
0174
0175     def skip_test(self, myarch):
0176         # If architecture not set always run test
0177         if self.arch == '':
0178             # log.warning("test for arch %s is ok" % myarch)
0179             return False
0180
0181         # Allow multiple values in assignment separated by ','
0182         arch_list = self.arch.split(',')
0183
0184         # Handle negated list such as !s390x,ppc
0185         if arch_list[0][0] == '!':
0186             arch_list[0] = arch_list[0][1:]
0187             log.warning("excluded architecture list %s" % arch_list)
0188             for arch_item in arch_list:
0189                 # log.warning("test for %s arch is %s" % (arch_item, myarch))
0190                 if arch_item == myarch:
0191                     return True
0192             return False
0193
0194         for arch_item in arch_list:
0195             # log.warning("test for architecture '%s' current '%s'" % (arch_item, myarch))
0196             if arch_item == myarch:
0197                 return False
0198         return True
0199
0200     def load_events(self, path, events):
0201         parser_event = configparser.SafeConfigParser()
0202         parser_event.read(path)
0203
0204         # The event record section header contains 'event' word,
0205         # optionaly followed by ':' allowing to load 'parent
0206         # event' first as a base
0207         for section in filter(self.is_event, parser_event.sections()):
0208
0209             parser_items = parser_event.items(section);
0210             base_items   = {}
0211
0212             # Read parent event if there's any
0213             if (':' in section):
0214                 base = section[section.index(':') + 1:]
0215                 parser_base = configparser.SafeConfigParser()
0216                 parser_base.read(self.test_dir + '/' + base)
0217                 base_items = parser_base.items('event')
0218
0219             e = Event(section, parser_items, base_items)
0220             events[section] = e
0221
0222     def run_cmd(self, tempdir):
0223         junk1, junk2, junk3, junk4, myarch = (os.uname())
0224
0225         if self.skip_test(myarch):
0226             raise Notest(self, myarch)
0227
0228         cmd = "PERF_TEST_ATTR=%s %s %s -o %s/perf.data %s" % (tempdir,
0229               self.perf, self.command, tempdir, self.args)
0230         ret = os.WEXITSTATUS(os.system(cmd))
0231
0232         log.info("  '%s' ret '%s', expected '%s'" % (cmd, str(ret), str(self.ret)))
0233
0234         if not data_equal(str(ret), str(self.ret)):
0235             raise Unsup(self)
0236
0237     def compare(self, expect, result):
0238         match = {}
0239
0240         log.debug("  compare");
0241
0242         # For each expected event find all matching
0243         # events in result. Fail if there's not any.
0244         for exp_name, exp_event in expect.items():
0245             exp_list = []
0246             res_event = {}
0247             log.debug("    matching [%s]" % exp_name)
0248             for res_name, res_event in result.items():
0249                 log.debug("      to [%s]" % res_name)
0250                 if (exp_event.equal(res_event)):
0251                     exp_list.append(res_name)
0252                     log.debug("    ->OK")
0253                 else:
0254                     log.debug("    ->FAIL");
0255
0256             log.debug("    match: [%s] matches %s" % (exp_name, str(exp_list)))
0257
0258             # we did not any matching event - fail
0259             if not exp_list:
0260                 if exp_event.optional():
0261                     log.debug("    %s does not match, but is optional" % exp_name)
0262                 else:
0263                     if not res_event:
0264                         log.debug("    res_event is empty");
0265                     else:
0266                         exp_event.diff(res_event)
0267                     raise Fail(self, 'match failure');
0268
0269             match[exp_name] = exp_list
0270
0271         # For each defined group in the expected events
0272         # check we match the same group in the result.
0273         for exp_name, exp_event in expect.items():
0274             group = exp_event.group
0275
0276             if (group == ''):
0277                 continue
0278
0279             for res_name in match[exp_name]:
0280                 res_group = result[res_name].group
0281                 if res_group not in match[group]:
0282                     raise Fail(self, 'group failure')
0283
0284                 log.debug("    group: [%s] matches group leader %s" %
0285                          (exp_name, str(match[group])))
0286
0287         log.debug("  matched")
0288
0289     def resolve_groups(self, events):
0290         for name, event in events.items():
0291             group_fd = event['group_fd'];
0292             if group_fd == '-1':
0293                 continue;
0294
0295             for iname, ievent in events.items():
0296                 if (ievent['fd'] == group_fd):
0297                     event.group = iname
0298                     log.debug('[%s] has group leader [%s]' % (name, iname))
0299                     break;
0300
0301     def run(self):
0302         tempdir = tempfile.mkdtemp();
0303
0304         try:
0305             # run the test script
0306             self.run_cmd(tempdir);
0307
0308             # load events expectation for the test
0309             log.debug("  loading result events");
0310             for f in glob.glob(tempdir + '/event*'):
0311                 self.load_events(f, self.result);
0312
0313             # resolve group_fd to event names
0314             self.resolve_groups(self.expect);
0315             self.resolve_groups(self.result);
0316
0317             # do the expectation - results matching - both ways
0318             self.compare(self.expect, self.result)
0319             self.compare(self.result, self.expect)
0320
0321         finally:
0322             # cleanup
0323             shutil.rmtree(tempdir)
0324
0325
0326 def run_tests(options):
0327     for f in glob.glob(options.test_dir + '/' + options.test):
0328         try:
0329             Test(f, options).run()
0330         except Unsup as obj:
0331             log.warning("unsupp  %s" % obj.getMsg())
0332         except Notest as obj:
0333             log.warning("skipped %s" % obj.getMsg())
0334
0335 def setup_log(verbose):
0336     global log
0337     level = logging.CRITICAL
0338
0339     if verbose == 1:
0340         level = logging.WARNING
0341     if verbose == 2:
0342         level = logging.INFO
0343     if verbose >= 3:
0344         level = logging.DEBUG
0345
0346     log = logging.getLogger('test')
0347     log.setLevel(level)
0348     ch  = logging.StreamHandler()
0349     ch.setLevel(level)
0350     formatter = logging.Formatter('%(message)s')
0351     ch.setFormatter(formatter)
0352     log.addHandler(ch)
0353
0354 USAGE = '''%s [OPTIONS]
0355   -d dir  # tests dir
0356   -p path # perf binary
0357   -t test # single test
0358   -v      # verbose level
0359 ''' % sys.argv[0]
0360
0361 def main():
0362     parser = optparse.OptionParser(usage=USAGE)
0363
0364     parser.add_option("-t", "--test",
0365                       action="store", type="string", dest="test")
0366     parser.add_option("-d", "--test-dir",
0367                       action="store", type="string", dest="test_dir")
0368     parser.add_option("-p", "--perf",
0369                       action="store", type="string", dest="perf")
0370     parser.add_option("-v", "--verbose",
0371                       default=0, action="count", dest="verbose")
0372
0373     options, args = parser.parse_args()
0374     if args:
0375         parser.error('FAILED wrong arguments %s' %  ' '.join(args))
0376         return -1
0377
0378     setup_log(options.verbose)
0379
0380     if not options.test_dir:
0381         print('FAILED no -d option specified')
0382         sys.exit(-1)
0383
0384     if not options.test:
0385         options.test = 'test*'
0386
0387     try:
0388         run_tests(options)
0389
0390     except Fail as obj:
0391         print("FAILED %s" % obj.getMsg())
0392         sys.exit(-1)
0393
0394     sys.exit(0)
0395
0396 if __name__ == '__main__':
0397     main()