0001
0002
0003
0004
0005
0006
0007
0008
0009
0010
0011
0012
0013 from __future__ import print_function
0014
0015 import os
0016 import sys
0017 import struct
0018 import datetime
0019
0020
0021
0022
0023
0024
0025
0026
0027
0028
0029
0030
0031
0032
0033
0034
0035
0036
0037
0038
0039
0040
0041
0042
0043
0044
0045
0046
0047
0048
0049
0050
0051
0052
0053
0054
0055
0056
0057
0058
0059
0060
0061
0062
0063
0064
0065
0066
0067
0068
0069
0070
0071
0072
0073
0074
0075
0076
0077
0078
0079
0080
0081
0082
0083
0084
0085
0086
0087
0088
0089
0090
0091
0092
0093
0094
0095
0096
0097
0098
0099
0100
0101
0102
0103
0104
0105
0106
0107
0108
0109
0110
0111
0112
0113
0114
0115
0116
0117
0118
0119
0120
0121
0122
0123
0124
0125
0126
0127
0128
0129
0130
0131
0132
0133
0134
0135
0136
0137
0138
0139
0140
0141
0142
0143
0144
0145
0146
0147
0148
0149
0150
0151
0152
0153
0154
0155
0156
0157
0158
0159
0160
0161
0162
0163
0164
0165
0166
0167
0168
0169
0170
0171
0172
0173
0174
0175
0176
0177
0178
0179
0180
0181
0182
0183
0184
0185
0186
0187
0188
0189
0190
0191
0192
0193
0194
0195
0196
0197
0198
0199
0200
0201
0202
0203
0204
0205
0206
0207
0208
0209
0210
0211
0212
0213
0214
0215 pyside_version_1 = True
0216 if not "pyside-version-1" in sys.argv:
0217 try:
0218 from PySide2.QtSql import *
0219 pyside_version_1 = False
0220 except:
0221 pass
0222
0223 if pyside_version_1:
0224 from PySide.QtSql import *
0225
0226 if sys.version_info < (3, 0):
0227 def toserverstr(str):
0228 return str
0229 def toclientstr(str):
0230 return str
0231 else:
0232
0233 def toserverstr(str):
0234 return bytes(str, "UTF_8")
0235 def toclientstr(str):
0236 return bytes(str, "UTF_8")
0237
0238
0239 from ctypes import *
0240 libpq = CDLL("libpq.so.5")
0241 PQconnectdb = libpq.PQconnectdb
0242 PQconnectdb.restype = c_void_p
0243 PQconnectdb.argtypes = [ c_char_p ]
0244 PQfinish = libpq.PQfinish
0245 PQfinish.argtypes = [ c_void_p ]
0246 PQstatus = libpq.PQstatus
0247 PQstatus.restype = c_int
0248 PQstatus.argtypes = [ c_void_p ]
0249 PQexec = libpq.PQexec
0250 PQexec.restype = c_void_p
0251 PQexec.argtypes = [ c_void_p, c_char_p ]
0252 PQresultStatus = libpq.PQresultStatus
0253 PQresultStatus.restype = c_int
0254 PQresultStatus.argtypes = [ c_void_p ]
0255 PQputCopyData = libpq.PQputCopyData
0256 PQputCopyData.restype = c_int
0257 PQputCopyData.argtypes = [ c_void_p, c_void_p, c_int ]
0258 PQputCopyEnd = libpq.PQputCopyEnd
0259 PQputCopyEnd.restype = c_int
0260 PQputCopyEnd.argtypes = [ c_void_p, c_void_p ]
0261
0262 sys.path.append(os.environ['PERF_EXEC_PATH'] + \
0263 '/scripts/python/Perf-Trace-Util/lib/Perf/Trace')
0264
0265
0266
0267
0268
0269 perf_db_export_mode = True
0270 perf_db_export_calls = False
0271 perf_db_export_callchains = False
0272
0273 def printerr(*args, **kw_args):
0274 print(*args, file=sys.stderr, **kw_args)
0275
0276 def printdate(*args, **kw_args):
0277 print(datetime.datetime.today(), *args, sep=' ', **kw_args)
0278
0279 def usage():
0280 printerr("Usage is: export-to-postgresql.py <database name> [<columns>] [<calls>] [<callchains>] [<pyside-version-1>]");
0281 printerr("where: columns 'all' or 'branches'");
0282 printerr(" calls 'calls' => create calls and call_paths table");
0283 printerr(" callchains 'callchains' => create call_paths table");
0284 printerr(" pyside-version-1 'pyside-version-1' => use pyside version 1");
0285 raise Exception("Too few or bad arguments")
0286
0287 if (len(sys.argv) < 2):
0288 usage()
0289
0290 dbname = sys.argv[1]
0291
0292 if (len(sys.argv) >= 3):
0293 columns = sys.argv[2]
0294 else:
0295 columns = "all"
0296
0297 if columns not in ("all", "branches"):
0298 usage()
0299
0300 branches = (columns == "branches")
0301
0302 for i in range(3,len(sys.argv)):
0303 if (sys.argv[i] == "calls"):
0304 perf_db_export_calls = True
0305 elif (sys.argv[i] == "callchains"):
0306 perf_db_export_callchains = True
0307 elif (sys.argv[i] == "pyside-version-1"):
0308 pass
0309 else:
0310 usage()
0311
0312 output_dir_name = os.getcwd() + "/" + dbname + "-perf-data"
0313 os.mkdir(output_dir_name)
0314
0315 def do_query(q, s):
0316 if (q.exec_(s)):
0317 return
0318 raise Exception("Query failed: " + q.lastError().text())
0319
0320 printdate("Creating database...")
0321
0322 db = QSqlDatabase.addDatabase('QPSQL')
0323 query = QSqlQuery(db)
0324 db.setDatabaseName('postgres')
0325 db.open()
0326 try:
0327 do_query(query, 'CREATE DATABASE ' + dbname)
0328 except:
0329 os.rmdir(output_dir_name)
0330 raise
0331 query.finish()
0332 query.clear()
0333 db.close()
0334
0335 db.setDatabaseName(dbname)
0336 db.open()
0337
0338 query = QSqlQuery(db)
0339 do_query(query, 'SET client_min_messages TO WARNING')
0340
0341 do_query(query, 'CREATE TABLE selected_events ('
0342 'id bigint NOT NULL,'
0343 'name varchar(80))')
0344 do_query(query, 'CREATE TABLE machines ('
0345 'id bigint NOT NULL,'
0346 'pid integer,'
0347 'root_dir varchar(4096))')
0348 do_query(query, 'CREATE TABLE threads ('
0349 'id bigint NOT NULL,'
0350 'machine_id bigint,'
0351 'process_id bigint,'
0352 'pid integer,'
0353 'tid integer)')
0354 do_query(query, 'CREATE TABLE comms ('
0355 'id bigint NOT NULL,'
0356 'comm varchar(16),'
0357 'c_thread_id bigint,'
0358 'c_time bigint,'
0359 'exec_flag boolean)')
0360 do_query(query, 'CREATE TABLE comm_threads ('
0361 'id bigint NOT NULL,'
0362 'comm_id bigint,'
0363 'thread_id bigint)')
0364 do_query(query, 'CREATE TABLE dsos ('
0365 'id bigint NOT NULL,'
0366 'machine_id bigint,'
0367 'short_name varchar(256),'
0368 'long_name varchar(4096),'
0369 'build_id varchar(64))')
0370 do_query(query, 'CREATE TABLE symbols ('
0371 'id bigint NOT NULL,'
0372 'dso_id bigint,'
0373 'sym_start bigint,'
0374 'sym_end bigint,'
0375 'binding integer,'
0376 'name varchar(2048))')
0377 do_query(query, 'CREATE TABLE branch_types ('
0378 'id integer NOT NULL,'
0379 'name varchar(80))')
0380
0381 if branches:
0382 do_query(query, 'CREATE TABLE samples ('
0383 'id bigint NOT NULL,'
0384 'evsel_id bigint,'
0385 'machine_id bigint,'
0386 'thread_id bigint,'
0387 'comm_id bigint,'
0388 'dso_id bigint,'
0389 'symbol_id bigint,'
0390 'sym_offset bigint,'
0391 'ip bigint,'
0392 'time bigint,'
0393 'cpu integer,'
0394 'to_dso_id bigint,'
0395 'to_symbol_id bigint,'
0396 'to_sym_offset bigint,'
0397 'to_ip bigint,'
0398 'branch_type integer,'
0399 'in_tx boolean,'
0400 'call_path_id bigint,'
0401 'insn_count bigint,'
0402 'cyc_count bigint,'
0403 'flags integer)')
0404 else:
0405 do_query(query, 'CREATE TABLE samples ('
0406 'id bigint NOT NULL,'
0407 'evsel_id bigint,'
0408 'machine_id bigint,'
0409 'thread_id bigint,'
0410 'comm_id bigint,'
0411 'dso_id bigint,'
0412 'symbol_id bigint,'
0413 'sym_offset bigint,'
0414 'ip bigint,'
0415 'time bigint,'
0416 'cpu integer,'
0417 'to_dso_id bigint,'
0418 'to_symbol_id bigint,'
0419 'to_sym_offset bigint,'
0420 'to_ip bigint,'
0421 'period bigint,'
0422 'weight bigint,'
0423 'transaction bigint,'
0424 'data_src bigint,'
0425 'branch_type integer,'
0426 'in_tx boolean,'
0427 'call_path_id bigint,'
0428 'insn_count bigint,'
0429 'cyc_count bigint,'
0430 'flags integer)')
0431
0432 if perf_db_export_calls or perf_db_export_callchains:
0433 do_query(query, 'CREATE TABLE call_paths ('
0434 'id bigint NOT NULL,'
0435 'parent_id bigint,'
0436 'symbol_id bigint,'
0437 'ip bigint)')
0438 if perf_db_export_calls:
0439 do_query(query, 'CREATE TABLE calls ('
0440 'id bigint NOT NULL,'
0441 'thread_id bigint,'
0442 'comm_id bigint,'
0443 'call_path_id bigint,'
0444 'call_time bigint,'
0445 'return_time bigint,'
0446 'branch_count bigint,'
0447 'call_id bigint,'
0448 'return_id bigint,'
0449 'parent_call_path_id bigint,'
0450 'flags integer,'
0451 'parent_id bigint,'
0452 'insn_count bigint,'
0453 'cyc_count bigint)')
0454
0455 do_query(query, 'CREATE TABLE ptwrite ('
0456 'id bigint NOT NULL,'
0457 'payload bigint,'
0458 'exact_ip boolean)')
0459
0460 do_query(query, 'CREATE TABLE cbr ('
0461 'id bigint NOT NULL,'
0462 'cbr integer,'
0463 'mhz integer,'
0464 'percent integer)')
0465
0466 do_query(query, 'CREATE TABLE mwait ('
0467 'id bigint NOT NULL,'
0468 'hints integer,'
0469 'extensions integer)')
0470
0471 do_query(query, 'CREATE TABLE pwre ('
0472 'id bigint NOT NULL,'
0473 'cstate integer,'
0474 'subcstate integer,'
0475 'hw boolean)')
0476
0477 do_query(query, 'CREATE TABLE exstop ('
0478 'id bigint NOT NULL,'
0479 'exact_ip boolean)')
0480
0481 do_query(query, 'CREATE TABLE pwrx ('
0482 'id bigint NOT NULL,'
0483 'deepest_cstate integer,'
0484 'last_cstate integer,'
0485 'wake_reason integer)')
0486
0487 do_query(query, 'CREATE TABLE context_switches ('
0488 'id bigint NOT NULL,'
0489 'machine_id bigint,'
0490 'time bigint,'
0491 'cpu integer,'
0492 'thread_out_id bigint,'
0493 'comm_out_id bigint,'
0494 'thread_in_id bigint,'
0495 'comm_in_id bigint,'
0496 'flags integer)')
0497
0498 do_query(query, 'CREATE VIEW machines_view AS '
0499 'SELECT '
0500 'id,'
0501 'pid,'
0502 'root_dir,'
0503 'CASE WHEN id=0 THEN \'unknown\' WHEN pid=-1 THEN \'host\' ELSE \'guest\' END AS host_or_guest'
0504 ' FROM machines')
0505
0506 do_query(query, 'CREATE VIEW dsos_view AS '
0507 'SELECT '
0508 'id,'
0509 'machine_id,'
0510 '(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest,'
0511 'short_name,'
0512 'long_name,'
0513 'build_id'
0514 ' FROM dsos')
0515
0516 do_query(query, 'CREATE VIEW symbols_view AS '
0517 'SELECT '
0518 'id,'
0519 'name,'
0520 '(SELECT short_name FROM dsos WHERE id=dso_id) AS dso,'
0521 'dso_id,'
0522 'sym_start,'
0523 'sym_end,'
0524 'CASE WHEN binding=0 THEN \'local\' WHEN binding=1 THEN \'global\' ELSE \'weak\' END AS binding'
0525 ' FROM symbols')
0526
0527 do_query(query, 'CREATE VIEW threads_view AS '
0528 'SELECT '
0529 'id,'
0530 'machine_id,'
0531 '(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest,'
0532 'process_id,'
0533 'pid,'
0534 'tid'
0535 ' FROM threads')
0536
0537 do_query(query, 'CREATE VIEW comm_threads_view AS '
0538 'SELECT '
0539 'comm_id,'
0540 '(SELECT comm FROM comms WHERE id = comm_id) AS command,'
0541 'thread_id,'
0542 '(SELECT pid FROM threads WHERE id = thread_id) AS pid,'
0543 '(SELECT tid FROM threads WHERE id = thread_id) AS tid'
0544 ' FROM comm_threads')
0545
0546 if perf_db_export_calls or perf_db_export_callchains:
0547 do_query(query, 'CREATE VIEW call_paths_view AS '
0548 'SELECT '
0549 'c.id,'
0550 'to_hex(c.ip) AS ip,'
0551 'c.symbol_id,'
0552 '(SELECT name FROM symbols WHERE id = c.symbol_id) AS symbol,'
0553 '(SELECT dso_id FROM symbols WHERE id = c.symbol_id) AS dso_id,'
0554 '(SELECT dso FROM symbols_view WHERE id = c.symbol_id) AS dso_short_name,'
0555 'c.parent_id,'
0556 'to_hex(p.ip) AS parent_ip,'
0557 'p.symbol_id AS parent_symbol_id,'
0558 '(SELECT name FROM symbols WHERE id = p.symbol_id) AS parent_symbol,'
0559 '(SELECT dso_id FROM symbols WHERE id = p.symbol_id) AS parent_dso_id,'
0560 '(SELECT dso FROM symbols_view WHERE id = p.symbol_id) AS parent_dso_short_name'
0561 ' FROM call_paths c INNER JOIN call_paths p ON p.id = c.parent_id')
0562 if perf_db_export_calls:
0563 do_query(query, 'CREATE VIEW calls_view AS '
0564 'SELECT '
0565 'calls.id,'
0566 'thread_id,'
0567 '(SELECT pid FROM threads WHERE id = thread_id) AS pid,'
0568 '(SELECT tid FROM threads WHERE id = thread_id) AS tid,'
0569 '(SELECT comm FROM comms WHERE id = comm_id) AS command,'
0570 'call_path_id,'
0571 'to_hex(ip) AS ip,'
0572 'symbol_id,'
0573 '(SELECT name FROM symbols WHERE id = symbol_id) AS symbol,'
0574 'call_time,'
0575 'return_time,'
0576 'return_time - call_time AS elapsed_time,'
0577 'branch_count,'
0578 'insn_count,'
0579 'cyc_count,'
0580 'CASE WHEN cyc_count=0 THEN CAST(0 AS NUMERIC(20, 2)) ELSE CAST((CAST(insn_count AS FLOAT) / cyc_count) AS NUMERIC(20, 2)) END AS IPC,'
0581 'call_id,'
0582 'return_id,'
0583 'CASE WHEN flags=0 THEN \'\' WHEN flags=1 THEN \'no call\' WHEN flags=2 THEN \'no return\' WHEN flags=3 THEN \'no call/return\' WHEN flags=6 THEN \'jump\' ELSE CAST ( flags AS VARCHAR(6) ) END AS flags,'
0584 'parent_call_path_id,'
0585 'calls.parent_id'
0586 ' FROM calls INNER JOIN call_paths ON call_paths.id = call_path_id')
0587
0588 do_query(query, 'CREATE VIEW samples_view AS '
0589 'SELECT '
0590 'id,'
0591 'time,'
0592 'cpu,'
0593 '(SELECT pid FROM threads WHERE id = thread_id) AS pid,'
0594 '(SELECT tid FROM threads WHERE id = thread_id) AS tid,'
0595 '(SELECT comm FROM comms WHERE id = comm_id) AS command,'
0596 '(SELECT name FROM selected_events WHERE id = evsel_id) AS event,'
0597 'to_hex(ip) AS ip_hex,'
0598 '(SELECT name FROM symbols WHERE id = symbol_id) AS symbol,'
0599 'sym_offset,'
0600 '(SELECT short_name FROM dsos WHERE id = dso_id) AS dso_short_name,'
0601 'to_hex(to_ip) AS to_ip_hex,'
0602 '(SELECT name FROM symbols WHERE id = to_symbol_id) AS to_symbol,'
0603 'to_sym_offset,'
0604 '(SELECT short_name FROM dsos WHERE id = to_dso_id) AS to_dso_short_name,'
0605 '(SELECT name FROM branch_types WHERE id = branch_type) AS branch_type_name,'
0606 'in_tx,'
0607 'insn_count,'
0608 'cyc_count,'
0609 'CASE WHEN cyc_count=0 THEN CAST(0 AS NUMERIC(20, 2)) ELSE CAST((CAST(insn_count AS FLOAT) / cyc_count) AS NUMERIC(20, 2)) END AS IPC,'
0610 'flags'
0611 ' FROM samples')
0612
0613 do_query(query, 'CREATE VIEW ptwrite_view AS '
0614 'SELECT '
0615 'ptwrite.id,'
0616 'time,'
0617 'cpu,'
0618 'to_hex(payload) AS payload_hex,'
0619 'CASE WHEN exact_ip=FALSE THEN \'False\' ELSE \'True\' END AS exact_ip'
0620 ' FROM ptwrite'
0621 ' INNER JOIN samples ON samples.id = ptwrite.id')
0622
0623 do_query(query, 'CREATE VIEW cbr_view AS '
0624 'SELECT '
0625 'cbr.id,'
0626 'time,'
0627 'cpu,'
0628 'cbr,'
0629 'mhz,'
0630 'percent'
0631 ' FROM cbr'
0632 ' INNER JOIN samples ON samples.id = cbr.id')
0633
0634 do_query(query, 'CREATE VIEW mwait_view AS '
0635 'SELECT '
0636 'mwait.id,'
0637 'time,'
0638 'cpu,'
0639 'to_hex(hints) AS hints_hex,'
0640 'to_hex(extensions) AS extensions_hex'
0641 ' FROM mwait'
0642 ' INNER JOIN samples ON samples.id = mwait.id')
0643
0644 do_query(query, 'CREATE VIEW pwre_view AS '
0645 'SELECT '
0646 'pwre.id,'
0647 'time,'
0648 'cpu,'
0649 'cstate,'
0650 'subcstate,'
0651 'CASE WHEN hw=FALSE THEN \'False\' ELSE \'True\' END AS hw'
0652 ' FROM pwre'
0653 ' INNER JOIN samples ON samples.id = pwre.id')
0654
0655 do_query(query, 'CREATE VIEW exstop_view AS '
0656 'SELECT '
0657 'exstop.id,'
0658 'time,'
0659 'cpu,'
0660 'CASE WHEN exact_ip=FALSE THEN \'False\' ELSE \'True\' END AS exact_ip'
0661 ' FROM exstop'
0662 ' INNER JOIN samples ON samples.id = exstop.id')
0663
0664 do_query(query, 'CREATE VIEW pwrx_view AS '
0665 'SELECT '
0666 'pwrx.id,'
0667 'time,'
0668 'cpu,'
0669 'deepest_cstate,'
0670 'last_cstate,'
0671 'CASE WHEN wake_reason=1 THEN \'Interrupt\''
0672 ' WHEN wake_reason=2 THEN \'Timer Deadline\''
0673 ' WHEN wake_reason=4 THEN \'Monitored Address\''
0674 ' WHEN wake_reason=8 THEN \'HW\''
0675 ' ELSE CAST ( wake_reason AS VARCHAR(2) )'
0676 'END AS wake_reason'
0677 ' FROM pwrx'
0678 ' INNER JOIN samples ON samples.id = pwrx.id')
0679
0680 do_query(query, 'CREATE VIEW power_events_view AS '
0681 'SELECT '
0682 'samples.id,'
0683 'samples.time,'
0684 'samples.cpu,'
0685 'selected_events.name AS event,'
0686 'FORMAT(\'%6s\', cbr.cbr) AS cbr,'
0687 'FORMAT(\'%6s\', cbr.mhz) AS MHz,'
0688 'FORMAT(\'%5s\', cbr.percent) AS percent,'
0689 'to_hex(mwait.hints) AS hints_hex,'
0690 'to_hex(mwait.extensions) AS extensions_hex,'
0691 'FORMAT(\'%3s\', pwre.cstate) AS cstate,'
0692 'FORMAT(\'%3s\', pwre.subcstate) AS subcstate,'
0693 'CASE WHEN pwre.hw=FALSE THEN \'False\' WHEN pwre.hw=TRUE THEN \'True\' ELSE NULL END AS hw,'
0694 'CASE WHEN exstop.exact_ip=FALSE THEN \'False\' WHEN exstop.exact_ip=TRUE THEN \'True\' ELSE NULL END AS exact_ip,'
0695 'FORMAT(\'%3s\', pwrx.deepest_cstate) AS deepest_cstate,'
0696 'FORMAT(\'%3s\', pwrx.last_cstate) AS last_cstate,'
0697 'CASE WHEN pwrx.wake_reason=1 THEN \'Interrupt\''
0698 ' WHEN pwrx.wake_reason=2 THEN \'Timer Deadline\''
0699 ' WHEN pwrx.wake_reason=4 THEN \'Monitored Address\''
0700 ' WHEN pwrx.wake_reason=8 THEN \'HW\''
0701 ' ELSE FORMAT(\'%2s\', pwrx.wake_reason)'
0702 'END AS wake_reason'
0703 ' FROM cbr'
0704 ' FULL JOIN mwait ON mwait.id = cbr.id'
0705 ' FULL JOIN pwre ON pwre.id = cbr.id'
0706 ' FULL JOIN exstop ON exstop.id = cbr.id'
0707 ' FULL JOIN pwrx ON pwrx.id = cbr.id'
0708 ' INNER JOIN samples ON samples.id = coalesce(cbr.id, mwait.id, pwre.id, exstop.id, pwrx.id)'
0709 ' INNER JOIN selected_events ON selected_events.id = samples.evsel_id'
0710 ' ORDER BY samples.id')
0711
0712 do_query(query, 'CREATE VIEW context_switches_view AS '
0713 'SELECT '
0714 'context_switches.id,'
0715 'context_switches.machine_id,'
0716 'context_switches.time,'
0717 'context_switches.cpu,'
0718 'th_out.pid AS pid_out,'
0719 'th_out.tid AS tid_out,'
0720 'comm_out.comm AS comm_out,'
0721 'th_in.pid AS pid_in,'
0722 'th_in.tid AS tid_in,'
0723 'comm_in.comm AS comm_in,'
0724 'CASE WHEN context_switches.flags = 0 THEN \'in\''
0725 ' WHEN context_switches.flags = 1 THEN \'out\''
0726 ' WHEN context_switches.flags = 3 THEN \'out preempt\''
0727 ' ELSE CAST ( context_switches.flags AS VARCHAR(11) )'
0728 'END AS flags'
0729 ' FROM context_switches'
0730 ' INNER JOIN threads AS th_out ON th_out.id = context_switches.thread_out_id'
0731 ' INNER JOIN threads AS th_in ON th_in.id = context_switches.thread_in_id'
0732 ' INNER JOIN comms AS comm_out ON comm_out.id = context_switches.comm_out_id'
0733 ' INNER JOIN comms AS comm_in ON comm_in.id = context_switches.comm_in_id')
0734
0735 file_header = struct.pack("!11sii", b"PGCOPY\n\377\r\n\0", 0, 0)
0736 file_trailer = b"\377\377"
0737
0738 def open_output_file(file_name):
0739 path_name = output_dir_name + "/" + file_name
0740 file = open(path_name, "wb+")
0741 file.write(file_header)
0742 return file
0743
0744 def close_output_file(file):
0745 file.write(file_trailer)
0746 file.close()
0747
0748 def copy_output_file_direct(file, table_name):
0749 close_output_file(file)
0750 sql = "COPY " + table_name + " FROM '" + file.name + "' (FORMAT 'binary')"
0751 do_query(query, sql)
0752
0753
0754 def copy_output_file(file, table_name):
0755 conn = PQconnectdb(toclientstr("dbname = " + dbname))
0756 if (PQstatus(conn)):
0757 raise Exception("COPY FROM STDIN PQconnectdb failed")
0758 file.write(file_trailer)
0759 file.seek(0)
0760 sql = "COPY " + table_name + " FROM STDIN (FORMAT 'binary')"
0761 res = PQexec(conn, toclientstr(sql))
0762 if (PQresultStatus(res) != 4):
0763 raise Exception("COPY FROM STDIN PQexec failed")
0764 data = file.read(65536)
0765 while (len(data)):
0766 ret = PQputCopyData(conn, data, len(data))
0767 if (ret != 1):
0768 raise Exception("COPY FROM STDIN PQputCopyData failed, error " + str(ret))
0769 data = file.read(65536)
0770 ret = PQputCopyEnd(conn, None)
0771 if (ret != 1):
0772 raise Exception("COPY FROM STDIN PQputCopyEnd failed, error " + str(ret))
0773 PQfinish(conn)
0774
0775 def remove_output_file(file):
0776 name = file.name
0777 file.close()
0778 os.unlink(name)
0779
0780 evsel_file = open_output_file("evsel_table.bin")
0781 machine_file = open_output_file("machine_table.bin")
0782 thread_file = open_output_file("thread_table.bin")
0783 comm_file = open_output_file("comm_table.bin")
0784 comm_thread_file = open_output_file("comm_thread_table.bin")
0785 dso_file = open_output_file("dso_table.bin")
0786 symbol_file = open_output_file("symbol_table.bin")
0787 branch_type_file = open_output_file("branch_type_table.bin")
0788 sample_file = open_output_file("sample_table.bin")
0789 if perf_db_export_calls or perf_db_export_callchains:
0790 call_path_file = open_output_file("call_path_table.bin")
0791 if perf_db_export_calls:
0792 call_file = open_output_file("call_table.bin")
0793 ptwrite_file = open_output_file("ptwrite_table.bin")
0794 cbr_file = open_output_file("cbr_table.bin")
0795 mwait_file = open_output_file("mwait_table.bin")
0796 pwre_file = open_output_file("pwre_table.bin")
0797 exstop_file = open_output_file("exstop_table.bin")
0798 pwrx_file = open_output_file("pwrx_table.bin")
0799 context_switches_file = open_output_file("context_switches_table.bin")
0800
0801 def trace_begin():
0802 printdate("Writing to intermediate files...")
0803
0804 evsel_table(0, "unknown")
0805 machine_table(0, 0, "unknown")
0806 thread_table(0, 0, 0, -1, -1)
0807 comm_table(0, "unknown", 0, 0, 0)
0808 dso_table(0, 0, "unknown", "unknown", "")
0809 symbol_table(0, 0, 0, 0, 0, "unknown")
0810 sample_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
0811 if perf_db_export_calls or perf_db_export_callchains:
0812 call_path_table(0, 0, 0, 0)
0813 call_return_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
0814
0815 unhandled_count = 0
0816
0817 def is_table_empty(table_name):
0818 do_query(query, 'SELECT * FROM ' + table_name + ' LIMIT 1');
0819 if query.next():
0820 return False
0821 return True
0822
0823 def drop(table_name):
0824 do_query(query, 'DROP VIEW ' + table_name + '_view');
0825 do_query(query, 'DROP TABLE ' + table_name);
0826
0827 def trace_end():
0828 printdate("Copying to database...")
0829 copy_output_file(evsel_file, "selected_events")
0830 copy_output_file(machine_file, "machines")
0831 copy_output_file(thread_file, "threads")
0832 copy_output_file(comm_file, "comms")
0833 copy_output_file(comm_thread_file, "comm_threads")
0834 copy_output_file(dso_file, "dsos")
0835 copy_output_file(symbol_file, "symbols")
0836 copy_output_file(branch_type_file, "branch_types")
0837 copy_output_file(sample_file, "samples")
0838 if perf_db_export_calls or perf_db_export_callchains:
0839 copy_output_file(call_path_file, "call_paths")
0840 if perf_db_export_calls:
0841 copy_output_file(call_file, "calls")
0842 copy_output_file(ptwrite_file, "ptwrite")
0843 copy_output_file(cbr_file, "cbr")
0844 copy_output_file(mwait_file, "mwait")
0845 copy_output_file(pwre_file, "pwre")
0846 copy_output_file(exstop_file, "exstop")
0847 copy_output_file(pwrx_file, "pwrx")
0848 copy_output_file(context_switches_file, "context_switches")
0849
0850 printdate("Removing intermediate files...")
0851 remove_output_file(evsel_file)
0852 remove_output_file(machine_file)
0853 remove_output_file(thread_file)
0854 remove_output_file(comm_file)
0855 remove_output_file(comm_thread_file)
0856 remove_output_file(dso_file)
0857 remove_output_file(symbol_file)
0858 remove_output_file(branch_type_file)
0859 remove_output_file(sample_file)
0860 if perf_db_export_calls or perf_db_export_callchains:
0861 remove_output_file(call_path_file)
0862 if perf_db_export_calls:
0863 remove_output_file(call_file)
0864 remove_output_file(ptwrite_file)
0865 remove_output_file(cbr_file)
0866 remove_output_file(mwait_file)
0867 remove_output_file(pwre_file)
0868 remove_output_file(exstop_file)
0869 remove_output_file(pwrx_file)
0870 remove_output_file(context_switches_file)
0871 os.rmdir(output_dir_name)
0872 printdate("Adding primary keys")
0873 do_query(query, 'ALTER TABLE selected_events ADD PRIMARY KEY (id)')
0874 do_query(query, 'ALTER TABLE machines ADD PRIMARY KEY (id)')
0875 do_query(query, 'ALTER TABLE threads ADD PRIMARY KEY (id)')
0876 do_query(query, 'ALTER TABLE comms ADD PRIMARY KEY (id)')
0877 do_query(query, 'ALTER TABLE comm_threads ADD PRIMARY KEY (id)')
0878 do_query(query, 'ALTER TABLE dsos ADD PRIMARY KEY (id)')
0879 do_query(query, 'ALTER TABLE symbols ADD PRIMARY KEY (id)')
0880 do_query(query, 'ALTER TABLE branch_types ADD PRIMARY KEY (id)')
0881 do_query(query, 'ALTER TABLE samples ADD PRIMARY KEY (id)')
0882 if perf_db_export_calls or perf_db_export_callchains:
0883 do_query(query, 'ALTER TABLE call_paths ADD PRIMARY KEY (id)')
0884 if perf_db_export_calls:
0885 do_query(query, 'ALTER TABLE calls ADD PRIMARY KEY (id)')
0886 do_query(query, 'ALTER TABLE ptwrite ADD PRIMARY KEY (id)')
0887 do_query(query, 'ALTER TABLE cbr ADD PRIMARY KEY (id)')
0888 do_query(query, 'ALTER TABLE mwait ADD PRIMARY KEY (id)')
0889 do_query(query, 'ALTER TABLE pwre ADD PRIMARY KEY (id)')
0890 do_query(query, 'ALTER TABLE exstop ADD PRIMARY KEY (id)')
0891 do_query(query, 'ALTER TABLE pwrx ADD PRIMARY KEY (id)')
0892 do_query(query, 'ALTER TABLE context_switches ADD PRIMARY KEY (id)')
0893
0894 printdate("Adding foreign keys")
0895 do_query(query, 'ALTER TABLE threads '
0896 'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id),'
0897 'ADD CONSTRAINT processfk FOREIGN KEY (process_id) REFERENCES threads (id)')
0898 do_query(query, 'ALTER TABLE comms '
0899 'ADD CONSTRAINT threadfk FOREIGN KEY (c_thread_id) REFERENCES threads (id)')
0900 do_query(query, 'ALTER TABLE comm_threads '
0901 'ADD CONSTRAINT commfk FOREIGN KEY (comm_id) REFERENCES comms (id),'
0902 'ADD CONSTRAINT threadfk FOREIGN KEY (thread_id) REFERENCES threads (id)')
0903 do_query(query, 'ALTER TABLE dsos '
0904 'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id)')
0905 do_query(query, 'ALTER TABLE symbols '
0906 'ADD CONSTRAINT dsofk FOREIGN KEY (dso_id) REFERENCES dsos (id)')
0907 do_query(query, 'ALTER TABLE samples '
0908 'ADD CONSTRAINT evselfk FOREIGN KEY (evsel_id) REFERENCES selected_events (id),'
0909 'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id),'
0910 'ADD CONSTRAINT threadfk FOREIGN KEY (thread_id) REFERENCES threads (id),'
0911 'ADD CONSTRAINT commfk FOREIGN KEY (comm_id) REFERENCES comms (id),'
0912 'ADD CONSTRAINT dsofk FOREIGN KEY (dso_id) REFERENCES dsos (id),'
0913 'ADD CONSTRAINT symbolfk FOREIGN KEY (symbol_id) REFERENCES symbols (id),'
0914 'ADD CONSTRAINT todsofk FOREIGN KEY (to_dso_id) REFERENCES dsos (id),'
0915 'ADD CONSTRAINT tosymbolfk FOREIGN KEY (to_symbol_id) REFERENCES symbols (id)')
0916 if perf_db_export_calls or perf_db_export_callchains:
0917 do_query(query, 'ALTER TABLE call_paths '
0918 'ADD CONSTRAINT parentfk FOREIGN KEY (parent_id) REFERENCES call_paths (id),'
0919 'ADD CONSTRAINT symbolfk FOREIGN KEY (symbol_id) REFERENCES symbols (id)')
0920 if perf_db_export_calls:
0921 do_query(query, 'ALTER TABLE calls '
0922 'ADD CONSTRAINT threadfk FOREIGN KEY (thread_id) REFERENCES threads (id),'
0923 'ADD CONSTRAINT commfk FOREIGN KEY (comm_id) REFERENCES comms (id),'
0924 'ADD CONSTRAINT call_pathfk FOREIGN KEY (call_path_id) REFERENCES call_paths (id),'
0925 'ADD CONSTRAINT callfk FOREIGN KEY (call_id) REFERENCES samples (id),'
0926 'ADD CONSTRAINT returnfk FOREIGN KEY (return_id) REFERENCES samples (id),'
0927 'ADD CONSTRAINT parent_call_pathfk FOREIGN KEY (parent_call_path_id) REFERENCES call_paths (id)')
0928 do_query(query, 'CREATE INDEX pcpid_idx ON calls (parent_call_path_id)')
0929 do_query(query, 'CREATE INDEX pid_idx ON calls (parent_id)')
0930 do_query(query, 'ALTER TABLE comms ADD has_calls boolean')
0931 do_query(query, 'UPDATE comms SET has_calls = TRUE WHERE comms.id IN (SELECT DISTINCT comm_id FROM calls)')
0932 do_query(query, 'ALTER TABLE ptwrite '
0933 'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)')
0934 do_query(query, 'ALTER TABLE cbr '
0935 'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)')
0936 do_query(query, 'ALTER TABLE mwait '
0937 'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)')
0938 do_query(query, 'ALTER TABLE pwre '
0939 'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)')
0940 do_query(query, 'ALTER TABLE exstop '
0941 'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)')
0942 do_query(query, 'ALTER TABLE pwrx '
0943 'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)')
0944 do_query(query, 'ALTER TABLE context_switches '
0945 'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id),'
0946 'ADD CONSTRAINT toutfk FOREIGN KEY (thread_out_id) REFERENCES threads (id),'
0947 'ADD CONSTRAINT tinfk FOREIGN KEY (thread_in_id) REFERENCES threads (id),'
0948 'ADD CONSTRAINT coutfk FOREIGN KEY (comm_out_id) REFERENCES comms (id),'
0949 'ADD CONSTRAINT cinfk FOREIGN KEY (comm_in_id) REFERENCES comms (id)')
0950
0951 printdate("Dropping unused tables")
0952 if is_table_empty("ptwrite"):
0953 drop("ptwrite")
0954 if is_table_empty("mwait") and is_table_empty("pwre") and is_table_empty("exstop") and is_table_empty("pwrx"):
0955 do_query(query, 'DROP VIEW power_events_view');
0956 drop("mwait")
0957 drop("pwre")
0958 drop("exstop")
0959 drop("pwrx")
0960 if is_table_empty("cbr"):
0961 drop("cbr")
0962 if is_table_empty("context_switches"):
0963 drop("context_switches")
0964
0965 if (unhandled_count):
0966 printdate("Warning: ", unhandled_count, " unhandled events")
0967 printdate("Done")
0968
0969 def trace_unhandled(event_name, context, event_fields_dict):
0970 global unhandled_count
0971 unhandled_count += 1
0972
0973 def sched__sched_switch(*x):
0974 pass
0975
0976 def evsel_table(evsel_id, evsel_name, *x):
0977 evsel_name = toserverstr(evsel_name)
0978 n = len(evsel_name)
0979 fmt = "!hiqi" + str(n) + "s"
0980 value = struct.pack(fmt, 2, 8, evsel_id, n, evsel_name)
0981 evsel_file.write(value)
0982
0983 def machine_table(machine_id, pid, root_dir, *x):
0984 root_dir = toserverstr(root_dir)
0985 n = len(root_dir)
0986 fmt = "!hiqiii" + str(n) + "s"
0987 value = struct.pack(fmt, 3, 8, machine_id, 4, pid, n, root_dir)
0988 machine_file.write(value)
0989
0990 def thread_table(thread_id, machine_id, process_id, pid, tid, *x):
0991 value = struct.pack("!hiqiqiqiiii", 5, 8, thread_id, 8, machine_id, 8, process_id, 4, pid, 4, tid)
0992 thread_file.write(value)
0993
0994 def comm_table(comm_id, comm_str, thread_id, time, exec_flag, *x):
0995 comm_str = toserverstr(comm_str)
0996 n = len(comm_str)
0997 fmt = "!hiqi" + str(n) + "s" + "iqiqiB"
0998 value = struct.pack(fmt, 5, 8, comm_id, n, comm_str, 8, thread_id, 8, time, 1, exec_flag)
0999 comm_file.write(value)
1000
1001 def comm_thread_table(comm_thread_id, comm_id, thread_id, *x):
1002 fmt = "!hiqiqiq"
1003 value = struct.pack(fmt, 3, 8, comm_thread_id, 8, comm_id, 8, thread_id)
1004 comm_thread_file.write(value)
1005
1006 def dso_table(dso_id, machine_id, short_name, long_name, build_id, *x):
1007 short_name = toserverstr(short_name)
1008 long_name = toserverstr(long_name)
1009 build_id = toserverstr(build_id)
1010 n1 = len(short_name)
1011 n2 = len(long_name)
1012 n3 = len(build_id)
1013 fmt = "!hiqiqi" + str(n1) + "si" + str(n2) + "si" + str(n3) + "s"
1014 value = struct.pack(fmt, 5, 8, dso_id, 8, machine_id, n1, short_name, n2, long_name, n3, build_id)
1015 dso_file.write(value)
1016
1017 def symbol_table(symbol_id, dso_id, sym_start, sym_end, binding, symbol_name, *x):
1018 symbol_name = toserverstr(symbol_name)
1019 n = len(symbol_name)
1020 fmt = "!hiqiqiqiqiii" + str(n) + "s"
1021 value = struct.pack(fmt, 6, 8, symbol_id, 8, dso_id, 8, sym_start, 8, sym_end, 4, binding, n, symbol_name)
1022 symbol_file.write(value)
1023
1024 def branch_type_table(branch_type, name, *x):
1025 name = toserverstr(name)
1026 n = len(name)
1027 fmt = "!hiii" + str(n) + "s"
1028 value = struct.pack(fmt, 2, 4, branch_type, n, name)
1029 branch_type_file.write(value)
1030
1031 def sample_table(sample_id, evsel_id, machine_id, thread_id, comm_id, dso_id, symbol_id, sym_offset, ip, time, cpu, to_dso_id, to_symbol_id, to_sym_offset, to_ip, period, weight, transaction, data_src, branch_type, in_tx, call_path_id, insn_cnt, cyc_cnt, flags, *x):
1032 if branches:
1033 value = struct.pack("!hiqiqiqiqiqiqiqiqiqiqiiiqiqiqiqiiiBiqiqiqii", 21, 8, sample_id, 8, evsel_id, 8, machine_id, 8, thread_id, 8, comm_id, 8, dso_id, 8, symbol_id, 8, sym_offset, 8, ip, 8, time, 4, cpu, 8, to_dso_id, 8, to_symbol_id, 8, to_sym_offset, 8, to_ip, 4, branch_type, 1, in_tx, 8, call_path_id, 8, insn_cnt, 8, cyc_cnt, 4, flags)
1034 else:
1035 value = struct.pack("!hiqiqiqiqiqiqiqiqiqiqiiiqiqiqiqiqiqiqiqiiiBiqiqiqii", 25, 8, sample_id, 8, evsel_id, 8, machine_id, 8, thread_id, 8, comm_id, 8, dso_id, 8, symbol_id, 8, sym_offset, 8, ip, 8, time, 4, cpu, 8, to_dso_id, 8, to_symbol_id, 8, to_sym_offset, 8, to_ip, 8, period, 8, weight, 8, transaction, 8, data_src, 4, branch_type, 1, in_tx, 8, call_path_id, 8, insn_cnt, 8, cyc_cnt, 4, flags)
1036 sample_file.write(value)
1037
1038 def call_path_table(cp_id, parent_id, symbol_id, ip, *x):
1039 fmt = "!hiqiqiqiq"
1040 value = struct.pack(fmt, 4, 8, cp_id, 8, parent_id, 8, symbol_id, 8, ip)
1041 call_path_file.write(value)
1042
1043 def call_return_table(cr_id, thread_id, comm_id, call_path_id, call_time, return_time, branch_count, call_id, return_id, parent_call_path_id, flags, parent_id, insn_cnt, cyc_cnt, *x):
1044 fmt = "!hiqiqiqiqiqiqiqiqiqiqiiiqiqiq"
1045 value = struct.pack(fmt, 14, 8, cr_id, 8, thread_id, 8, comm_id, 8, call_path_id, 8, call_time, 8, return_time, 8, branch_count, 8, call_id, 8, return_id, 8, parent_call_path_id, 4, flags, 8, parent_id, 8, insn_cnt, 8, cyc_cnt)
1046 call_file.write(value)
1047
1048 def ptwrite(id, raw_buf):
1049 data = struct.unpack_from("<IQ", raw_buf)
1050 flags = data[0]
1051 payload = data[1]
1052 exact_ip = flags & 1
1053 value = struct.pack("!hiqiqiB", 3, 8, id, 8, payload, 1, exact_ip)
1054 ptwrite_file.write(value)
1055
1056 def cbr(id, raw_buf):
1057 data = struct.unpack_from("<BBBBII", raw_buf)
1058 cbr = data[0]
1059 MHz = (data[4] + 500) / 1000
1060 percent = ((cbr * 1000 / data[2]) + 5) / 10
1061 value = struct.pack("!hiqiiiiii", 4, 8, id, 4, cbr, 4, int(MHz), 4, int(percent))
1062 cbr_file.write(value)
1063
1064 def mwait(id, raw_buf):
1065 data = struct.unpack_from("<IQ", raw_buf)
1066 payload = data[1]
1067 hints = payload & 0xff
1068 extensions = (payload >> 32) & 0x3
1069 value = struct.pack("!hiqiiii", 3, 8, id, 4, hints, 4, extensions)
1070 mwait_file.write(value)
1071
1072 def pwre(id, raw_buf):
1073 data = struct.unpack_from("<IQ", raw_buf)
1074 payload = data[1]
1075 hw = (payload >> 7) & 1
1076 cstate = (payload >> 12) & 0xf
1077 subcstate = (payload >> 8) & 0xf
1078 value = struct.pack("!hiqiiiiiB", 4, 8, id, 4, cstate, 4, subcstate, 1, hw)
1079 pwre_file.write(value)
1080
1081 def exstop(id, raw_buf):
1082 data = struct.unpack_from("<I", raw_buf)
1083 flags = data[0]
1084 exact_ip = flags & 1
1085 value = struct.pack("!hiqiB", 2, 8, id, 1, exact_ip)
1086 exstop_file.write(value)
1087
1088 def pwrx(id, raw_buf):
1089 data = struct.unpack_from("<IQ", raw_buf)
1090 payload = data[1]
1091 deepest_cstate = payload & 0xf
1092 last_cstate = (payload >> 4) & 0xf
1093 wake_reason = (payload >> 8) & 0xf
1094 value = struct.pack("!hiqiiiiii", 4, 8, id, 4, deepest_cstate, 4, last_cstate, 4, wake_reason)
1095 pwrx_file.write(value)
1096
1097 def synth_data(id, config, raw_buf, *x):
1098 if config == 0:
1099 ptwrite(id, raw_buf)
1100 elif config == 1:
1101 mwait(id, raw_buf)
1102 elif config == 2:
1103 pwre(id, raw_buf)
1104 elif config == 3:
1105 exstop(id, raw_buf)
1106 elif config == 4:
1107 pwrx(id, raw_buf)
1108 elif config == 5:
1109 cbr(id, raw_buf)
1110
1111 def context_switch_table(id, machine_id, time, cpu, thread_out_id, comm_out_id, thread_in_id, comm_in_id, flags, *x):
1112 fmt = "!hiqiqiqiiiqiqiqiqii"
1113 value = struct.pack(fmt, 9, 8, id, 8, machine_id, 8, time, 4, cpu, 8, thread_out_id, 8, comm_out_id, 8, thread_in_id, 8, comm_in_id, 4, flags)
1114 context_switches_file.write(value)