ui/static/stagepage.js

0001 /*
0002  * Licensed to the Apache Software Foundation (ASF) under one or more
0003  * contributor license agreements.  See the NOTICE file distributed with
0004  * this work for additional information regarding copyright ownership.
0005  * The ASF licenses this file to You under the Apache License, Version 2.0
0006  * (the "License"); you may not use this file except in compliance with
0007  * the License.  You may obtain a copy of the License at
0008  *
0009  *    http://www.apache.org/licenses/LICENSE-2.0
0010  *
0011  * Unless required by applicable law or agreed to in writing, software
0012  * distributed under the License is distributed on an "AS IS" BASIS,
0013  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0014  * See the License for the specific language governing permissions and
0015  * limitations under the License.
0016  */
0017
0018 var shouldBlockUI = true;
0019
0020 $(document).ajaxStop(function () {
0021     if (shouldBlockUI) {
0022         $.unblockUI();
0023         shouldBlockUI = false;
0024     }
0025 });
0026
0027 $(document).ajaxStart(function () {
0028     if (shouldBlockUI) {
0029         $.blockUI({message: '<h3>Loading Stage Page...</h3>'});
0030     }
0031 });
0032
0033 $.extend( $.fn.dataTable.ext.type.order, {
0034     "duration-pre": ConvertDurationString,
0035
0036     "duration-asc": function ( a, b ) {
0037         a = ConvertDurationString( a );
0038         b = ConvertDurationString( b );
0039         return ((a < b) ? -1 : ((a > b) ? 1 : 0));
0040     },
0041
0042     "duration-desc": function ( a, b ) {
0043         a = ConvertDurationString( a );
0044         b = ConvertDurationString( b );
0045         return ((a < b) ? 1 : ((a > b) ? -1 : 0));
0046     }
0047 } );
0048
0049 // This function will only parse the URL under certain format
0050 // e.g. (history) https://domain:50509/history/application_1536254569791_3806251/1/stages/stage/?id=4&attempt=1
0051 // e.g. (proxy) https://domain:50505/proxy/application_1502220952225_59143/stages/stage?id=4&attempt=1
0052 function stageEndPoint(appId) {
0053     var queryString = document.baseURI.split('?');
0054     var words = document.baseURI.split('/');
0055     var indexOfProxy = words.indexOf("proxy");
0056     var stageId = queryString[1].split("&").filter(word => word.includes("id="))[0].split("=")[1];
0057     if (indexOfProxy > 0) {
0058         var appId = words[indexOfProxy + 1];
0059         var newBaseURI = words.slice(0, words.indexOf("proxy") + 2).join('/');
0060         return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
0061     }
0062     var indexOfHistory = words.indexOf("history");
0063     if (indexOfHistory > 0) {
0064         var appId = words[indexOfHistory + 1];
0065         var appAttemptId = words[indexOfHistory + 2];
0066         var newBaseURI = words.slice(0, words.indexOf("history")).join('/');
0067         if (isNaN(appAttemptId) || appAttemptId == "0") {
0068             return newBaseURI + "/api/v1/applications/" + appId + "/stages/" + stageId;
0069         } else {
0070             return newBaseURI + "/api/v1/applications/" + appId + "/" + appAttemptId + "/stages/" + stageId;
0071         }
0072     }
0073     return location.origin + "/api/v1/applications/" + appId + "/stages/" + stageId;
0074 }
0075
0076 function getColumnNameForTaskMetricSummary(columnKey) {
0077     switch(columnKey) {
0078         case "executorRunTime":
0079             return "Duration";
0080
0081         case "jvmGcTime":
0082             return "GC Time";
0083
0084         case "gettingResultTime":
0085             return "Getting Result Time";
0086
0087         case "inputMetrics":
0088             return "Input Size / Records";
0089
0090         case "outputMetrics":
0091             return "Output Size / Records";
0092
0093         case "peakExecutionMemory":
0094             return "Peak Execution Memory";
0095
0096         case "resultSerializationTime":
0097             return "Result Serialization Time";
0098
0099         case "schedulerDelay":
0100             return "Scheduler Delay";
0101
0102         case "diskBytesSpilled":
0103             return "Spill (disk)";
0104
0105         case "memoryBytesSpilled":
0106             return "Spill (memory)";
0107
0108         case "shuffleReadMetrics":
0109             return "Shuffle Read Size / Records";
0110
0111         case "shuffleWriteMetrics":
0112             return "Shuffle Write Size / Records";
0113
0114         case "executorDeserializeTime":
0115             return "Task Deserialization Time";
0116
0117         case "shuffleReadBlockedTime":
0118             return "Shuffle Read Blocked Time";
0119
0120         case "shuffleRemoteReads":
0121             return "Shuffle Remote Reads";
0122
0123         default:
0124             return "NA";
0125     }
0126 }
0127
0128 function displayRowsForSummaryMetricsTable(row, type, columnIndex) {
0129     switch(row.columnKey) {
0130         case 'inputMetrics':
0131             var str = formatBytes(row.data.bytesRead[columnIndex], type) + " / " +
0132               row.data.recordsRead[columnIndex];
0133             return str;
0134             break;
0135
0136         case 'outputMetrics':
0137             var str = formatBytes(row.data.bytesWritten[columnIndex], type) + " / " +
0138               row.data.recordsWritten[columnIndex];
0139             return str;
0140             break;
0141
0142         case 'shuffleReadMetrics':
0143             var str = formatBytes(row.data.readBytes[columnIndex], type) + " / " +
0144               row.data.readRecords[columnIndex];
0145             return str;
0146             break;
0147
0148         case 'shuffleReadBlockedTime':
0149             var str = formatDuration(row.data.fetchWaitTime[columnIndex]);
0150             return str;
0151             break;
0152
0153         case 'shuffleRemoteReads':
0154             var str = formatBytes(row.data.remoteBytesRead[columnIndex], type);
0155             return str;
0156             break;
0157
0158         case 'shuffleWriteMetrics':
0159             var str = formatBytes(row.data.writeBytes[columnIndex], type) + " / " +
0160               row.data.writeRecords[columnIndex];
0161             return str;
0162             break;
0163
0164         default:
0165             return (row.columnKey == 'peakExecutionMemory' || row.columnKey == 'memoryBytesSpilled'
0166                     || row.columnKey == 'diskBytesSpilled') ? formatBytes(
0167                     row.data[columnIndex], type) : (formatDuration(row.data[columnIndex]));
0168
0169     }
0170 }
0171
0172 function createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTable) {
0173     var taskMetricsTable = "#summary-metrics-table";
0174     if ($.fn.dataTable.isDataTable(taskMetricsTable)) {
0175         taskSummaryMetricsDataTable.clear().draw();
0176         taskSummaryMetricsDataTable.rows.add(taskSummaryMetricsTable).draw();
0177     } else {
0178         var taskConf = {
0179             "data": taskSummaryMetricsTable,
0180             "columns": [
0181                 {data : 'metric'},
0182                 // Min
0183                 {
0184                     data: function (row, type) {
0185                         return displayRowsForSummaryMetricsTable(row, type, 0);
0186                     }
0187                 },
0188                 // 25th percentile
0189                 {
0190                     data: function (row, type) {
0191                         return displayRowsForSummaryMetricsTable(row, type, 1);
0192                     }
0193                 },
0194                 // Median
0195                 {
0196                     data: function (row, type) {
0197                         return displayRowsForSummaryMetricsTable(row, type, 2);
0198                     }
0199                 },
0200                 // 75th percentile
0201                 {
0202                     data: function (row, type) {
0203                         return displayRowsForSummaryMetricsTable(row, type, 3);
0204                     }
0205                 },
0206                 // Max
0207                 {
0208                     data: function (row, type) {
0209                         return displayRowsForSummaryMetricsTable(row, type, 4);
0210                     }
0211                 }
0212             ],
0213             "columnDefs": [
0214                 { "type": "duration", "targets": 1 },
0215                 { "type": "duration", "targets": 2 },
0216                 { "type": "duration", "targets": 3 },
0217                 { "type": "duration", "targets": 4 },
0218                 { "type": "duration", "targets": 5 }
0219             ],
0220             "paging": false,
0221             "searching": false,
0222             "order": [[0, "asc"]],
0223             "bSort": false,
0224             "bAutoWidth": false,
0225             "oLanguage": {
0226                 "sEmptyTable": "No tasks have reported metrics yet"
0227             }
0228         };
0229         taskSummaryMetricsDataTable = $(taskMetricsTable).DataTable(taskConf);
0230     }
0231     taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTable.slice();
0232 }
0233
0234 function createRowMetadataForColumn(colKey, data, checkboxId) {
0235   var row = {
0236       "metric": getColumnNameForTaskMetricSummary(colKey),
0237       "data": data,
0238       "checkboxId": checkboxId,
0239       "columnKey": colKey
0240   };
0241   return row;
0242 }
0243
0244 function reselectCheckboxesBasedOnTaskTableState() {
0245     var allChecked = true;
0246     var taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
0247     if (typeof taskTableSelector !== 'undefined' && taskSummaryMetricsTableCurrentStateArray.length > 0) {
0248         for (var k = 0; k < optionalColumns.length; k++) {
0249             if (taskTableSelector.column(optionalColumns[k]).visible()) {
0250                 $("#box-"+optionalColumns[k]).prop('checked', true);
0251                 taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == optionalColumns[k])[0]);
0252                 taskSummaryMetricsTableCurrentFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
0253             } else {
0254                 allChecked = false;
0255             }
0256         }
0257         if (allChecked) {
0258             $("#box-0").prop('checked', true);
0259         }
0260         createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableCurrentFilteredArray);
0261     }
0262 }
0263
0264 function getStageAttemptId() {
0265   var words = document.baseURI.split('?');
0266   var digitsRegex = /[0-9]+/;
0267   // We are using regex here to extract the stage attempt id as there might be certain url's with format
0268   // like /proxy/application_1539986433979_27115/stages/stage/?id=0&attempt=0#tasksTitle
0269   var stgAttemptId = words[1].split("&").filter(
0270       word => word.includes("attempt="))[0].split("=")[1].match(digitsRegex);
0271   return stgAttemptId;
0272 }
0273
0274 var taskSummaryMetricsTableArray = [];
0275 var taskSummaryMetricsTableCurrentStateArray = [];
0276 var taskSummaryMetricsDataTable;
0277 var optionalColumns = [11, 12, 13, 14, 15, 16, 17];
0278 var taskTableSelector;
0279
0280 $(document).ready(function () {
0281     setDataTableDefaults();
0282
0283     $("#showAdditionalMetrics").append(
0284         "<div><a id='additionalMetrics'>" +
0285         "<span class='expand-input-rate-arrow arrow-closed' id='arrowtoggle1'></span>" +
0286         " Show Additional Metrics" +
0287         "</a></div>" +
0288         "<div class='container-fluid container-fluid-div' id='toggle-metrics' hidden>" +
0289         "<div id='select_all' class='select-all-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-0' data-column='0'> Select All</div>" +
0290         "<div id='scheduler_delay' class='scheduler-delay-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-11' data-column='11'> Scheduler Delay</div>" +
0291         "<div id='task_deserialization_time' class='task-deserialization-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-12' data-column='12'> Task Deserialization Time</div>" +
0292         "<div id='shuffle_read_blocked_time' class='shuffle-read-blocked-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-13' data-column='13'> Shuffle Read Blocked Time</div>" +
0293         "<div id='shuffle_remote_reads' class='shuffle-remote-reads-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-14' data-column='14'> Shuffle Remote Reads</div>" +
0294         "<div id='result_serialization_time' class='result-serialization-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-15' data-column='15'> Result Serialization Time</div>" +
0295         "<div id='getting_result_time' class='getting-result-time-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-16' data-column='16'> Getting Result Time</div>" +
0296         "<div id='peak_execution_memory' class='peak-execution-memory-checkbox-div'><input type='checkbox' class='toggle-vis' id='box-17' data-column='17'> Peak Execution Memory</div>" +
0297         "</div>");
0298
0299     $('#scheduler_delay').attr("data-toggle", "tooltip")
0300         .attr("data-placement", "top")
0301         .attr("title", "Scheduler delay includes time to ship the task from the scheduler to the executor, and time to send " +
0302             "the task result from the executor to the scheduler. If scheduler delay is large, consider decreasing the size of tasks or decreasing the size of task results.");
0303     $('#task_deserialization_time').attr("data-toggle", "tooltip")
0304         .attr("data-placement", "top")
0305         .attr("title", "Time spent deserializing the task closure on the executor, including the time to read the broadcasted task.");
0306     $('#shuffle_read_blocked_time').attr("data-toggle", "tooltip")
0307         .attr("data-placement", "top")
0308         .attr("title", "Time that the task spent blocked waiting for shuffle data to be read from remote machines.");
0309     $('#shuffle_remote_reads').attr("data-toggle", "tooltip")
0310         .attr("data-placement", "top")
0311         .attr("title", "Total shuffle bytes read from remote executors. This is a subset of the shuffle read bytes; the remaining shuffle data is read locally. ");
0312     $('#result_serialization_time').attr("data-toggle", "tooltip")
0313             .attr("data-placement", "top")
0314             .attr("title", "Time spent serializing the task result on the executor before sending it back to the driver.");
0315     $('#getting_result_time').attr("data-toggle", "tooltip")
0316             .attr("data-placement", "top")
0317             .attr("title", "Time that the driver spends fetching task results from workers. If this is large, consider decreasing the amount of data returned from each task.");
0318     $('#peak_execution_memory').attr("data-toggle", "tooltip")
0319             .attr("data-placement", "top")
0320             .attr("title", "Execution memory refers to the memory used by internal data structures created during " +
0321                 "shuffles, aggregations and joins when Tungsten is enabled. The value of this accumulator " +
0322                 "should be approximately the sum of the peak sizes across all such data structures created " +
0323                 "in this task. For SQL jobs, this only tracks all unsafe operators, broadcast joins, and " +
0324                 "external sort.");
0325     $('[data-toggle="tooltip"]').tooltip();
0326     var tasksSummary = $("#parent-container");
0327     getStandAloneAppId(function (appId) {
0328         // rendering the UI page
0329         $.get(createTemplateURI(appId, "stagespage"), function(template) {
0330           tasksSummary.append(Mustache.render($(template).filter("#stages-summary-template").html()));
0331
0332           $("#additionalMetrics").click(function(){
0333               $("#arrowtoggle1").toggleClass("arrow-open arrow-closed");
0334               $("#toggle-metrics").toggle();
0335               if (window.localStorage) {
0336                   window.localStorage.setItem("arrowtoggle1class", $("#arrowtoggle1").attr('class'));
0337               }
0338           });
0339
0340           $("#aggregatedMetrics").click(function(){
0341               $("#arrowtoggle2").toggleClass("arrow-open arrow-closed");
0342               $("#toggle-aggregatedMetrics").toggle();
0343               if (window.localStorage) {
0344                   window.localStorage.setItem("arrowtoggle2class", $("#arrowtoggle2").attr('class'));
0345               }
0346           });
0347
0348         var endPoint = stageEndPoint(appId);
0349         var stageAttemptId = getStageAttemptId();
0350         $.getJSON(endPoint + "/" + stageAttemptId, function(response, status, jqXHR) {
0351
0352             var responseBody = response;
0353             var dataToShow = {};
0354             dataToShow.showInputData = responseBody.inputBytes > 0;
0355             dataToShow.showOutputData = responseBody.outputBytes > 0;
0356             dataToShow.showShuffleReadData = responseBody.shuffleReadBytes > 0;
0357             dataToShow.showShuffleWriteData = responseBody.shuffleWriteBytes > 0;
0358             dataToShow.showBytesSpilledData =
0359                 (responseBody.diskBytesSpilled > 0 || responseBody.memoryBytesSpilled > 0);
0360
0361             if (!dataToShow.showShuffleReadData) {
0362                 $('#shuffle_read_blocked_time').remove();
0363                 $('#shuffle_remote_reads').remove();
0364                 optionalColumns.splice(2, 2);
0365             }
0366
0367             // prepare data for executor summary table
0368             var stageExecutorSummaryInfoKeys = Object.keys(responseBody.executorSummary);
0369             $.getJSON(createRESTEndPointForExecutorsPage(appId),
0370               function(executorSummaryResponse, status, jqXHR) {
0371                 var executorDetailsMap = {};
0372                 executorSummaryResponse.forEach(function (executorDetail) {
0373                     executorDetailsMap[executorDetail.id] = executorDetail;
0374                 });
0375
0376                 var executorSummaryTable = [];
0377                 stageExecutorSummaryInfoKeys.forEach(function (columnKeyIndex) {
0378                     var executorSummary = responseBody.executorSummary[columnKeyIndex];
0379                     var executorDetail = executorDetailsMap[columnKeyIndex.toString()];
0380                     executorSummary.id = columnKeyIndex;
0381                     executorSummary.executorLogs = {};
0382                     executorSummary.hostPort = "CANNOT FIND ADDRESS";
0383
0384                     if (executorDetail) {
0385                         if (executorDetail["executorLogs"]) {
0386                             responseBody.executorSummary[columnKeyIndex].executorLogs =
0387                                 executorDetail["executorLogs"];
0388                             }
0389                         if (executorDetail["hostPort"]) {
0390                             responseBody.executorSummary[columnKeyIndex].hostPort =
0391                                 executorDetail["hostPort"];
0392                         }
0393                     }
0394                     executorSummaryTable.push(responseBody.executorSummary[columnKeyIndex]);
0395                 });
0396                 // building task aggregated metrics by executor table
0397                 var executorSummaryConf = {
0398                     "data": executorSummaryTable,
0399                     "columns": [
0400                         {data : "id"},
0401                         {data : "executorLogs", render: formatLogsCells},
0402                         {data : "hostPort"},
0403                         {
0404                             data : function (row, type) {
0405                                 return type === 'display' ? formatDuration(row.taskTime) : row.taskTime;
0406                             }
0407                         },
0408                         {
0409                             data : function (row, type) {
0410                                 var totaltasks = row.succeededTasks + row.failedTasks + row.killedTasks;
0411                                 return type === 'display' ? totaltasks : totaltasks.toString();
0412                             }
0413                         },
0414                         {data : "failedTasks"},
0415                         {data : "killedTasks"},
0416                         {data : "succeededTasks"},
0417                         {data : "isBlacklistedForStage"},
0418                         {
0419                             data : function (row, type) {
0420                                 return row.inputRecords != 0 ? formatBytes(row.inputBytes, type) + " / " + row.inputRecords : "";
0421                             }
0422                         },
0423                         {
0424                             data : function (row, type) {
0425                                 return row.outputRecords != 0 ? formatBytes(row.outputBytes, type) + " / " + row.outputRecords : "";
0426                             }
0427                         },
0428                         {
0429                             data : function (row, type) {
0430                                 return row.shuffleReadRecords != 0 ? formatBytes(row.shuffleRead, type) + " / " + row.shuffleReadRecords : "";
0431                             }
0432                         },
0433                         {
0434                             data : function (row, type) {
0435                                 return row.shuffleWriteRecords != 0 ? formatBytes(row.shuffleWrite, type) + " / " + row.shuffleWriteRecords : "";
0436                             }
0437                         },
0438                         {
0439                             data : function (row, type) {
0440                                 return typeof row.memoryBytesSpilled != 'undefined' ? formatBytes(row.memoryBytesSpilled, type) : "";
0441                             }
0442                         },
0443                         {
0444                             data : function (row, type) {
0445                                 return typeof row.diskBytesSpilled != 'undefined' ? formatBytes(row.diskBytesSpilled, type) : "";
0446                             }
0447                         }
0448                     ],
0449                     "order": [[0, "asc"]],
0450                     "bAutoWidth": false,
0451                     "oLanguage": {
0452                         "sEmptyTable": "No data to show yet"
0453                     }
0454                 };
0455                 var executorSummaryTableSelector =
0456                     $("#summary-executor-table").DataTable(executorSummaryConf);
0457                 $('#parent-container [data-toggle="tooltip"]').tooltip();
0458
0459                 executorSummaryTableSelector.column(9).visible(dataToShow.showInputData);
0460                 if (dataToShow.showInputData) {
0461                     $('#executor-summary-input').attr("data-toggle", "tooltip")
0462                         .attr("data-placement", "top")
0463                         .attr("title", "Bytes and records read from Hadoop or from Spark storage.");
0464                     $('#executor-summary-input').tooltip(true);
0465                 }
0466                 executorSummaryTableSelector.column(10).visible(dataToShow.showOutputData);
0467                 if (dataToShow.showOutputData) {
0468                     $('#executor-summary-output').attr("data-toggle", "tooltip")
0469                         .attr("data-placement", "top")
0470                         .attr("title", "Bytes and records written to Hadoop.");
0471                     $('#executor-summary-output').tooltip(true);
0472                 }
0473                 executorSummaryTableSelector.column(11).visible(dataToShow.showShuffleReadData);
0474                 if (dataToShow.showShuffleReadData) {
0475                     $('#executor-summary-shuffle-read').attr("data-toggle", "tooltip")
0476                         .attr("data-placement", "top")
0477                         .attr("title", "Total shuffle bytes and records read (includes both data read locally and data read from remote executors).");
0478                     $('#executor-summary-shuffle-read').tooltip(true);
0479                 }
0480                 executorSummaryTableSelector.column(12).visible(dataToShow.showShuffleWriteData);
0481                 if (dataToShow.showShuffleWriteData) {
0482                     $('#executor-summary-shuffle-write').attr("data-toggle", "tooltip")
0483                         .attr("data-placement", "top")
0484                         .attr("title", "Bytes and records written to disk in order to be read by a shuffle in a future stage.");
0485                     $('#executor-summary-shuffle-write').tooltip(true);
0486                 }
0487                 executorSummaryTableSelector.column(13).visible(dataToShow.showBytesSpilledData);
0488                 executorSummaryTableSelector.column(14).visible(dataToShow.showBytesSpilledData);
0489             });
0490
0491             // prepare data for accumulatorUpdates
0492             var accumulatorTable = responseBody.accumulatorUpdates.filter(accumUpdate =>
0493                 !(accumUpdate.name).toString().includes("internal."));
0494
0495                 var quantiles = "0,0.25,0.5,0.75,1.0";
0496                 $.getJSON(endPoint + "/" + stageAttemptId + "/taskSummary?quantiles=" + quantiles,
0497                   function(taskMetricsResponse, status, jqXHR) {
0498                     var taskMetricKeys = Object.keys(taskMetricsResponse);
0499                     taskMetricKeys.forEach(function (columnKey) {
0500                         switch(columnKey) {
0501                             case "shuffleReadMetrics":
0502                                 var row1 = createRowMetadataForColumn(
0503                                     columnKey, taskMetricsResponse[columnKey], 3);
0504                                 var row2 = createRowMetadataForColumn(
0505                                     "shuffleReadBlockedTime", taskMetricsResponse[columnKey], 13);
0506                                 var row3 = createRowMetadataForColumn(
0507                                     "shuffleRemoteReads", taskMetricsResponse[columnKey], 14);
0508                                 if (dataToShow.showShuffleReadData) {
0509                                     taskSummaryMetricsTableArray.push(row1);
0510                                     taskSummaryMetricsTableArray.push(row2);
0511                                     taskSummaryMetricsTableArray.push(row3);
0512                                 }
0513                                 break;
0514
0515                             case "schedulerDelay":
0516                                 var row = createRowMetadataForColumn(
0517                                     columnKey, taskMetricsResponse[columnKey], 11);
0518                                 taskSummaryMetricsTableArray.push(row);
0519                                 break;
0520
0521                             case "executorDeserializeTime":
0522                                 var row = createRowMetadataForColumn(
0523                                     columnKey, taskMetricsResponse[columnKey], 12);
0524                                 taskSummaryMetricsTableArray.push(row);
0525                                 break;
0526
0527                             case "resultSerializationTime":
0528                                 var row = createRowMetadataForColumn(
0529                                     columnKey, taskMetricsResponse[columnKey], 15);
0530                                 taskSummaryMetricsTableArray.push(row);
0531                                 break;
0532
0533                             case "gettingResultTime":
0534                                 var row = createRowMetadataForColumn(
0535                                     columnKey, taskMetricsResponse[columnKey], 16);
0536                                 taskSummaryMetricsTableArray.push(row);
0537                                 break;
0538
0539                             case "peakExecutionMemory":
0540                                 var row = createRowMetadataForColumn(
0541                                     columnKey, taskMetricsResponse[columnKey], 17);
0542                                 taskSummaryMetricsTableArray.push(row);
0543                                 break;
0544
0545                             case "inputMetrics":
0546                                 var row = createRowMetadataForColumn(
0547                                     columnKey, taskMetricsResponse[columnKey], 1);
0548                                 if (dataToShow.showInputData) {
0549                                     taskSummaryMetricsTableArray.push(row);
0550                                 }
0551                                 break;
0552
0553                             case "outputMetrics":
0554                                 var row = createRowMetadataForColumn(
0555                                     columnKey, taskMetricsResponse[columnKey], 2);
0556                                 if (dataToShow.showOutputData) {
0557                                     taskSummaryMetricsTableArray.push(row);
0558                                 }
0559                                 break;
0560
0561                             case "shuffleWriteMetrics":
0562                                 var row = createRowMetadataForColumn(
0563                                     columnKey, taskMetricsResponse[columnKey], 4);
0564                                 if (dataToShow.showShuffleWriteData) {
0565                                     taskSummaryMetricsTableArray.push(row);
0566                                 }
0567                                 break;
0568
0569                             case "diskBytesSpilled":
0570                                 var row = createRowMetadataForColumn(
0571                                     columnKey, taskMetricsResponse[columnKey], 5);
0572                                 if (dataToShow.showBytesSpilledData) {
0573                                     taskSummaryMetricsTableArray.push(row);
0574                                 }
0575                                 break;
0576
0577                             case "memoryBytesSpilled":
0578                                 var row = createRowMetadataForColumn(
0579                                     columnKey, taskMetricsResponse[columnKey], 6);
0580                                 if (dataToShow.showBytesSpilledData) {
0581                                     taskSummaryMetricsTableArray.push(row);
0582                                 }
0583                                 break;
0584
0585                             default:
0586                                 if (getColumnNameForTaskMetricSummary(columnKey) != "NA") {
0587                                     var row = createRowMetadataForColumn(
0588                                         columnKey, taskMetricsResponse[columnKey], 0);
0589                                     taskSummaryMetricsTableArray.push(row);
0590                                 }
0591                                 break;
0592                         }
0593                     });
0594                     var taskSummaryMetricsTableFilteredArray =
0595                         taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11);
0596                     taskSummaryMetricsTableCurrentStateArray = taskSummaryMetricsTableFilteredArray.slice();
0597                     reselectCheckboxesBasedOnTaskTableState();
0598                 });
0599
0600                 // building accumulator update table
0601                 var accumulatorConf = {
0602                     "data": accumulatorTable,
0603                     "columns": [
0604                         {data : "id"},
0605                         {data : "name"},
0606                         {data : "value"}
0607                     ],
0608                     "paging": false,
0609                     "searching": false,
0610                     "order": [[0, "asc"]],
0611                     "bAutoWidth": false
0612                 };
0613                 $("#accumulator-table").DataTable(accumulatorConf);
0614
0615                 // building tasks table that uses server side functionality
0616                 var totalTasksToShow = responseBody.numCompleteTasks + responseBody.numActiveTasks +
0617                     responseBody.numKilledTasks + responseBody.numFailedTasks;
0618                 var taskTable = "#active-tasks-table";
0619                 var taskConf = {
0620                     "serverSide": true,
0621                     "paging": true,
0622                     "info": true,
0623                     "processing": true,
0624                     "lengthMenu": [[20, 40, 60, 100, totalTasksToShow], [20, 40, 60, 100, "All"]],
0625                     "orderMulti": false,
0626                     "bAutoWidth": false,
0627                     "ajax": {
0628                         "url": endPoint + "/" + stageAttemptId + "/taskTable",
0629                         "data": function (data) {
0630                             var columnIndexToSort = 0;
0631                             var columnNameToSort = "Index";
0632                             if (data.order[0].column && data.order[0].column != "") {
0633                                 columnIndexToSort = parseInt(data.order[0].column);
0634                                 columnNameToSort = data.columns[columnIndexToSort].name;
0635                             }
0636                             delete data.columns;
0637                             data.numTasks = totalTasksToShow;
0638                             data.columnIndexToSort = columnIndexToSort;
0639                             data.columnNameToSort = columnNameToSort;
0640                         },
0641                         "dataSrc": function (jsons) {
0642                             var jsonStr = JSON.stringify(jsons);
0643                             var tasksToShow = JSON.parse(jsonStr);
0644                             return tasksToShow.aaData;
0645                         },
0646                         "error": function (jqXHR, textStatus, errorThrown) {
0647                             alert("Unable to connect to the server. Looks like the Spark " +
0648                               "application must have ended. Please Switch to the history UI.");
0649                             $("#active-tasks-table_processing").css("display","none");
0650                         }
0651                     },
0652                     "columns": [
0653                         {data: function (row, type) {
0654                             return type !== 'display' ? (isNaN(row.index) ? 0 : row.index ) : row.index;
0655                             },
0656                             name: "Index"
0657                         },
0658                         {data : "taskId", name: "ID"},
0659                         {data : "attempt", name: "Attempt"},
0660                         {data : "status", name: "Status"},
0661                         {data : "taskLocality", name: "Locality Level"},
0662                         {data : "executorId", name: "Executor ID"},
0663                         {data : "host", name: "Host"},
0664                         {data : "executorLogs", name: "Logs", render: formatLogsCells},
0665                         {data : "launchTime", name: "Launch Time", render: formatDate},
0666                         {
0667                             data : function (row, type) {
0668                                 if (row.taskMetrics && row.taskMetrics.executorRunTime) {
0669                                     return type === 'display' ? formatDuration(row.taskMetrics.executorRunTime) : row.taskMetrics.executorRunTime;
0670                                 } else {
0671                                     return "";
0672                                 }
0673                             },
0674                             name: "Duration"
0675                         },
0676                         {
0677                             data : function (row, type) {
0678                                 if (row.taskMetrics && row.taskMetrics.jvmGcTime) {
0679                                     return type === 'display' ? formatDuration(row.taskMetrics.jvmGcTime) : row.taskMetrics.jvmGcTime;
0680                                 } else {
0681                                     return "";
0682                                 }
0683                             },
0684                             name: "GC Time"
0685                         },
0686                         {
0687                             data : function (row, type) {
0688                                 if (row.schedulerDelay) {
0689                                     return type === 'display' ? formatDuration(row.schedulerDelay) : row.schedulerDelay;
0690                                 } else {
0691                                     return "";
0692                                 }
0693                             },
0694                             name: "Scheduler Delay"
0695                         },
0696                         {
0697                             data : function (row, type) {
0698                                 if (row.taskMetrics && row.taskMetrics.executorDeserializeTime) {
0699                                     return type === 'display' ? formatDuration(row.taskMetrics.executorDeserializeTime) : row.taskMetrics.executorDeserializeTime;
0700                                 } else {
0701                                     return "";
0702                                 }
0703                             },
0704                             name: "Task Deserialization Time"
0705                         },
0706                         {
0707                             data : function (row, type) {
0708                                 if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) {
0709                                     return type === 'display' ? formatDuration(row.taskMetrics.shuffleReadMetrics.fetchWaitTime) : row.taskMetrics.shuffleReadMetrics.fetchWaitTime;
0710                                 } else {
0711                                     return "";
0712                                 }
0713                             },
0714                             name: "Shuffle Read Blocked Time"
0715                         },
0716                         {
0717                             data : function (row, type) {
0718                                 if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics) {
0719                                     return type === 'display' ? formatBytes(row.taskMetrics.shuffleReadMetrics.remoteBytesRead, type) : row.taskMetrics.shuffleReadMetrics.remoteBytesRead;
0720                                 } else {
0721                                     return "";
0722                                 }
0723                             },
0724                             name: "Shuffle Remote Reads"
0725                         },
0726                         {
0727                             data : function (row, type) {
0728                                 if (row.taskMetrics && row.taskMetrics.resultSerializationTime) {
0729                                     return type === 'display' ? formatDuration(row.taskMetrics.resultSerializationTime) : row.taskMetrics.resultSerializationTime;
0730                                 } else {
0731                                     return "";
0732                                 }
0733                             },
0734                             name: "Result Serialization Time"
0735                         },
0736                         {
0737                             data : function (row, type) {
0738                                 if (row.gettingResultTime) {
0739                                     return type === 'display' ? formatDuration(row.gettingResultTime) : row.gettingResultTime;
0740                                 } else {
0741                                     return "";
0742                                 }
0743                             },
0744                             name: "Getting Result Time"
0745                         },
0746                         {
0747                             data : function (row, type) {
0748                                 if (row.taskMetrics && row.taskMetrics.peakExecutionMemory) {
0749                                     return type === 'display' ? formatBytes(row.taskMetrics.peakExecutionMemory, type) : row.taskMetrics.peakExecutionMemory;
0750                                 } else {
0751                                     return "";
0752                                 }
0753                             },
0754                             name: "Peak Execution Memory"
0755                         },
0756                         {
0757                             data : function (row, type) {
0758                                 if (accumulatorTable.length > 0 && row.accumulatorUpdates.length > 0) {
0759                                     var allAccums = "";
0760                                     row.accumulatorUpdates.forEach(function(accumulator) {
0761                                         allAccums += accumulator.name + ': ' + accumulator.update + "<BR>";
0762                                     })
0763                                     return allAccums;
0764                                 } else {
0765                                     return "";
0766                                 }
0767                             },
0768                             name: "Accumulators"
0769                         },
0770                         {
0771                             data : function (row, type) {
0772                                 if (row.taskMetrics && row.taskMetrics.inputMetrics && row.taskMetrics.inputMetrics.bytesRead > 0) {
0773                                     if (type === 'display') {
0774                                         return formatBytes(row.taskMetrics.inputMetrics.bytesRead, type) + " / " + row.taskMetrics.inputMetrics.recordsRead;
0775                                     } else {
0776                                         return row.taskMetrics.inputMetrics.bytesRead + " / " + row.taskMetrics.inputMetrics.recordsRead;
0777                                     }
0778                                 } else {
0779                                     return "";
0780                                 }
0781                             },
0782                             name: "Input Size / Records"
0783                         },
0784                         {
0785                             data : function (row, type) {
0786                                 if (row.taskMetrics && row.taskMetrics.outputMetrics && row.taskMetrics.outputMetrics.bytesWritten > 0) {
0787                                     if (type === 'display') {
0788                                         return formatBytes(row.taskMetrics.outputMetrics.bytesWritten, type) + " / " + row.taskMetrics.outputMetrics.recordsWritten;
0789                                     } else {
0790                                         return row.taskMetrics.outputMetrics.bytesWritten + " / " + row.taskMetrics.outputMetrics.recordsWritten;
0791                                     }
0792                                 } else {
0793                                     return "";
0794                                 }
0795                             },
0796                             name: "Output Size / Records"
0797                         },
0798                         {
0799                             data : function (row, type) {
0800                                 if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.writeTime > 0) {
0801                                     return type === 'display' ? formatDuration(parseInt(row.taskMetrics.shuffleWriteMetrics.writeTime) / 1000000) : row.taskMetrics.shuffleWriteMetrics.writeTime;
0802                                 } else {
0803                                     return "";
0804                                 }
0805                             },
0806                             name: "Write Time"
0807                         },
0808                         {
0809                             data : function (row, type) {
0810                                 if (row.taskMetrics && row.taskMetrics.shuffleWriteMetrics && row.taskMetrics.shuffleWriteMetrics.bytesWritten > 0) {
0811                                     if (type === 'display') {
0812                                         return formatBytes(row.taskMetrics.shuffleWriteMetrics.bytesWritten, type) + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten;
0813                                     } else {
0814                                         return row.taskMetrics.shuffleWriteMetrics.bytesWritten + " / " + row.taskMetrics.shuffleWriteMetrics.recordsWritten;
0815                                     }
0816                                 } else {
0817                                     return "";
0818                                 }
0819                             },
0820                             name: "Shuffle Write Size / Records"
0821                         },
0822                         {
0823                             data : function (row, type) {
0824                                 if (row.taskMetrics && row.taskMetrics.shuffleReadMetrics && row.taskMetrics.shuffleReadMetrics.localBytesRead > 0) {
0825                                     var totalBytesRead = parseInt(row.taskMetrics.shuffleReadMetrics.localBytesRead) + parseInt(row.taskMetrics.shuffleReadMetrics.remoteBytesRead);
0826                                     if (type === 'display') {
0827                                         return formatBytes(totalBytesRead, type) + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;
0828                                     } else {
0829                                         return totalBytesRead + " / " + row.taskMetrics.shuffleReadMetrics.recordsRead;
0830                                     }
0831                                 } else {
0832                                     return "";
0833                                 }
0834                             },
0835                             name: "Shuffle Read Size / Records"
0836                         },
0837                         {
0838                             data : function (row, type) {
0839                                 if (row.taskMetrics && row.taskMetrics.memoryBytesSpilled && row.taskMetrics.memoryBytesSpilled > 0) {
0840                                     return type === 'display' ? formatBytes(row.taskMetrics.memoryBytesSpilled, type) : row.taskMetrics.memoryBytesSpilled;
0841                                 } else {
0842                                     return "";
0843                                 }
0844                             },
0845                             name: "Spill (Memory)"
0846                         },
0847                         {
0848                             data : function (row, type) {
0849                                 if (row.taskMetrics && row.taskMetrics.diskBytesSpilled && row.taskMetrics.diskBytesSpilled > 0) {
0850                                     return type === 'display' ? formatBytes(row.taskMetrics.diskBytesSpilled, type) : row.taskMetrics.diskBytesSpilled;
0851                                 } else {
0852                                     return "";
0853                                 }
0854                             },
0855                             name: "Spill (Disk)"
0856                         },
0857                         {
0858                             data : function (row, type) {
0859                                 var msg = row.errorMessage;
0860                                 if (typeof msg === 'undefined') {
0861                                     return "";
0862                                 } else {
0863                                     var formHead = msg.substring(0, msg.indexOf("at"));
0864                                     var form = "<span onclick=\"this.parentNode.querySelector('.stacktrace-details').classList.toggle('collapsed')\" class=\"expand-details\">+details</span>";
0865                                     var formMsg = "<div class=\"stacktrace-details collapsed\"><pre>" + row.errorMessage + "</pre></div>";
0866                                     return formHead + form + formMsg;
0867                                 }
0868                             },
0869                             name: "Errors"
0870                         }
0871                     ],
0872                     "columnDefs": [
0873                         { "visible": false, "targets": 11 },
0874                         { "visible": false, "targets": 12 },
0875                         { "visible": false, "targets": 13 },
0876                         { "visible": false, "targets": 14 },
0877                         { "visible": false, "targets": 15 },
0878                         { "visible": false, "targets": 16 },
0879                         { "visible": false, "targets": 17 },
0880                         { "visible": false, "targets": 18 }
0881                     ],
0882                     "deferRender": true
0883                 };
0884                 taskTableSelector = $(taskTable).DataTable(taskConf);
0885                 $('#active-tasks-table_filter input').unbind();
0886                 var searchEvent;
0887                 $('#active-tasks-table_filter input').bind('keyup', function(e) {
0888                   if (typeof searchEvent !== 'undefined') {
0889                     window.clearTimeout(searchEvent);
0890                   }
0891                   var value = this.value;
0892                   searchEvent = window.setTimeout(function(){
0893                     taskTableSelector.search( value ).draw();}, 500);
0894                 });
0895                 reselectCheckboxesBasedOnTaskTableState();
0896
0897                 // hide or show columns dynamically event
0898                 $('input.toggle-vis').on('click', function(e){
0899                     // Get the column
0900                     var para = $(this).attr('data-column');
0901                     if (para == "0") {
0902                         var allColumns = taskTableSelector.columns(optionalColumns);
0903                         if ($(this).is(":checked")) {
0904                             $(".toggle-vis").prop('checked', true);
0905                             allColumns.visible(true);
0906                             createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableArray);
0907                         } else {
0908                             $(".toggle-vis").prop('checked', false);
0909                             allColumns.visible(false);
0910                             var taskSummaryMetricsTableFilteredArray =
0911                                 taskSummaryMetricsTableArray.filter(row => row.checkboxId < 11);
0912                             createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
0913                         }
0914                     } else {
0915                         var column = taskTableSelector.column(para);
0916                         // Toggle the visibility
0917                         column.visible(!column.visible());
0918                         var taskSummaryMetricsTableFilteredArray = [];
0919                         if ($(this).is(":checked")) {
0920                             taskSummaryMetricsTableCurrentStateArray.push(taskSummaryMetricsTableArray.filter(row => (row.checkboxId).toString() == para)[0]);
0921                             taskSummaryMetricsTableFilteredArray = taskSummaryMetricsTableCurrentStateArray.slice();
0922                         } else {
0923                             taskSummaryMetricsTableFilteredArray =
0924                                 taskSummaryMetricsTableCurrentStateArray.filter(row => (row.checkboxId).toString() != para);
0925                         }
0926                         createDataTableForTaskSummaryMetricsTable(taskSummaryMetricsTableFilteredArray);
0927                     }
0928                 });
0929
0930                 // title number and toggle list
0931                 $("#summaryMetricsTitle").html("Summary Metrics for " + "<a href='#tasksTitle'>" + responseBody.numCompleteTasks + " Completed Tasks" + "</a>");
0932                 $("#tasksTitle").html("Tasks (" + totalTasksToShow + ")");
0933
0934                 // hide or show the accumulate update table
0935                 if (accumulatorTable.length == 0) {
0936                     $("#accumulator-update-table").hide();
0937                 } else {
0938                     taskTableSelector.column(18).visible(true);
0939                     $("#accumulator-update-table").show();
0940                 }
0941                 // Showing relevant stage data depending on stage type for task table and executor
0942                 // summary table
0943                 taskTableSelector.column(19).visible(dataToShow.showInputData);
0944                 taskTableSelector.column(20).visible(dataToShow.showOutputData);
0945                 taskTableSelector.column(21).visible(dataToShow.showShuffleWriteData);
0946                 taskTableSelector.column(22).visible(dataToShow.showShuffleWriteData);
0947                 taskTableSelector.column(23).visible(dataToShow.showShuffleReadData);
0948                 taskTableSelector.column(24).visible(dataToShow.showBytesSpilledData);
0949                 taskTableSelector.column(25).visible(dataToShow.showBytesSpilledData);
0950
0951                 if (window.localStorage) {
0952                     if (window.localStorage.getItem("arrowtoggle1class") !== null &&
0953                         window.localStorage.getItem("arrowtoggle1class").includes("arrow-open")) {
0954                         $("#arrowtoggle1").toggleClass("arrow-open arrow-closed");
0955                         $("#toggle-metrics").toggle();
0956                     }
0957                     if (window.localStorage.getItem("arrowtoggle2class") !== null &&
0958                         window.localStorage.getItem("arrowtoggle2class").includes("arrow-open")) {
0959                         $("#arrowtoggle2").toggleClass("arrow-open arrow-closed");
0960                         $("#toggle-aggregatedMetrics").toggle();
0961                     }
0962                 }
0963             });
0964         });
0965     });
0966 });