Add tracking of parent depth level in order to show parent/child relationships (#45)

Add tracking of parent depth level in order to show parent/child relationships and more closely match Perfetto UI. Also fix various scalar data types that can come across as optional / nullable.

Author: ivberg

PerfettoCds/Pipeline/CompositeDataCookers/PerfettoCpuSchedEventCooker.cs

@@ -52,15 +52,15 @@ public void OnDataAvailable(IDataExtensionRetrieval requiredData)
             // those respective tables
             var joined = from schedSlice in schedSliceData
                          join thread in threadData on schedSlice.Utid equals thread.Utid
-                         join process in processData on thread.Upid equals process.Upid
+                         join process in processData on thread.Upid equals process.Upid into pd from process in pd.DefaultIfEmpty()
                          select new { schedSlice, thread, process };
 
             // Create events out of the joined results
             foreach (var result in joined)
             {
                 PerfettoCpuSchedEvent ev = new PerfettoCpuSchedEvent
                 (
-                    result.process.Name,
+                    result.process?.Name,
                     result.thread.Name,
                     new TimestampDelta(result.schedSlice.Duration),
                     new Timestamp(result.schedSlice.RelativeTimestamp),

PerfettoCds/Pipeline/CompositeDataCookers/PerfettoFtraceEventCooker.cs

@@ -63,7 +63,7 @@ public void OnDataAvailable(IDataExtensionRetrieval requiredData)
             // Thread and process info is contained in their respective tables
             var joined = from raw in rawData
                          join thread in threadData on raw.Utid equals thread.Utid
-                         join process in processData on thread.Upid equals process.Upid
+                         join process in processData on thread.Upid equals process.Upid into pd from process in pd.DefaultIfEmpty()
                          join arg in argsData on raw.ArgSetId equals arg.ArgSetId into args
                          select new { raw, args, thread, process };
 
@@ -104,7 +104,7 @@ join arg in argsData on raw.ArgSetId equals arg.ArgSetId into args
                 PerfettoFtraceEvent ev = new PerfettoFtraceEvent
                 (
                     new Timestamp(result.raw.RelativeTimestamp),
-                    result.process.Name,
+                    result.process?.Name,
                     result.thread.Name,
                     result.raw.Cpu,
                     result.raw.Name,

PerfettoCds/Pipeline/CompositeDataCookers/PerfettoGenericEventCooker.cs

@@ -2,18 +2,19 @@
 // Licensed under the MIT License.
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
 using System.Linq;
 using System.Reflection;
+using System.Xml;
+using System.Xml.Serialization;
 using Microsoft.Performance.SDK;
 using Microsoft.Performance.SDK.Extensibility;
 using Microsoft.Performance.SDK.Extensibility.DataCooking;
 using Microsoft.Performance.SDK.Processing;
 using PerfettoCds.Pipeline.DataOutput;
-using PerfettoProcessor;
-using System.Xml;
-using System.Xml.Serialization;
-using System.IO;
 using PerfettoCds.Pipeline.SourceDataCookers;
+using PerfettoProcessor;
 
 namespace PerfettoCds.Pipeline.CompositeDataCookers
 {
@@ -169,6 +170,9 @@ join processTrack in processTrackData on slice.TrackId equals processTrack.Id in
                          join process in processData on processTrack?.Upid equals process.Upid into pd2 from process in pd2.DefaultIfEmpty()
                          select new { slice, args, threadTrack, thread, threadProcess, process };
 
+            var longestRelTS = joined.Max(f => f.slice?.RelativeTimestamp);
+            var longestEndTs = longestRelTS.HasValue ? new Timestamp(longestRelTS.Value) : Timestamp.MaxValue;
+
             // Create events out of the joined results
             foreach (var result in joined)
             {
@@ -230,20 +234,37 @@ join processTrack in processTrackData on slice.TrackId equals processTrack.Id in
                     processName = $"{result.process.Name} {result.process.Pid}";
                 }
 
+                int parentTreeDepthLevel = 0;
+                long? currentParentId = result.slice.ParentId;
+                
+                // Walk the parent tree
+                while (currentParentId.HasValue)
+                {
+                    var parentPerfettoSliceEvent = sliceData[(int) currentParentId.Value];
+                    // Debug.Assert(parentPerfettoSliceEvent == null || (parentPerfettoSliceEvent.Id == currentParentId.Value)); // Should be guaranteed by slice Id ordering. Since we are relying on index being the Id
+                    currentParentId = parentPerfettoSliceEvent != null ? parentPerfettoSliceEvent.ParentId : null;
+                    parentTreeDepthLevel++;
+                }
+
                 PerfettoGenericEvent ev = new PerfettoGenericEvent
                 (
                    result.slice.Name,
                    result.slice.Type,
                    new TimestampDelta(result.slice.Duration),
                    new Timestamp(result.slice.RelativeTimestamp),
-                   new Timestamp(result.slice.RelativeTimestamp + result.slice.Duration),
+                   result.slice.Duration >= 0 ?             // Duration can be not complete / negative
+                    new Timestamp(result.slice.RelativeTimestamp + result.slice.Duration) :
+                    longestEndTs,
                    result.slice.Category,
                    result.slice.ArgSetId,
                    values,
                    argKeys,
                    processName,
                    threadName,
-                   provider
+                   provider,
+                   result.threadTrack,
+                   result.slice.ParentId,
+                   parentTreeDepthLevel
                 );
                 this.GenericEvents.AddEvent(ev);
             }

PerfettoCds/Pipeline/DataOutput/PerfettoGenericEvent.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 using Microsoft.Performance.SDK;
+using PerfettoProcessor;
 using System.Collections.Generic;
 
 namespace PerfettoCds.Pipeline.DataOutput
@@ -34,6 +35,12 @@ public readonly struct PerfettoGenericEvent
 
         public string Provider { get; }
 
+        public long? ParentId{ get; }
+
+        public int ParentTreeDepthLevel { get; }
+
+        public PerfettoThreadTrackEvent ThreadTrack { get; }
+
         public PerfettoGenericEvent(string eventName, 
             string type, 
             TimestampDelta duration, 
@@ -45,20 +52,26 @@ public PerfettoGenericEvent(string eventName,
             List<string> argKeys,
             string process,
             string thread,
-            string provider)
+            string provider,
+            PerfettoThreadTrackEvent threadTrack,
+            long? parentId,
+            int parentTreeDepthLevel)
         {
-            this.EventName = eventName;
-            this.Type = type;
-            this.Duration = duration;
-            this.StartTimestamp = startTimestamp;
-            this.EndTimestamp = endTimestamp;
-            this.Category = category;
-            this.ArgSetId = argSetId;
-            this.Values = values;
-            this.ArgKeys = argKeys;
-            this.Process = process;
-            this.Thread = thread;
-            this.Provider = provider;
+            EventName = eventName;
+            Type = type;
+            Duration = duration;
+            StartTimestamp = startTimestamp;
+            EndTimestamp = endTimestamp;
+            Category = category;
+            ArgSetId = argSetId;
+            Values = values;
+            ArgKeys = argKeys;
+            Process = process;
+            Thread = thread;
+            Provider = provider;
+            ThreadTrack = threadTrack;
+            ParentId = parentId;
+            ParentTreeDepthLevel = parentTreeDepthLevel;
         }
     }
 }

PerfettoCds/Pipeline/Tables/PerfettoGenericEventTable.cs

@@ -52,7 +52,7 @@ public class PerfettoGenericEventTable
             new UIHints { 
                 Width = 70,
                 AggregationMode = AggregationMode.Max,
-                SortPriority = 0,
+                SortPriority = 1,
                 SortOrder = SortOrder.Descending,
                 CellFormat = TimestampFormatter.FormatMillisecondsGrouped,
             });
@@ -78,6 +78,23 @@ public class PerfettoGenericEventTable
             new ColumnMetadata(new Guid("{e7d08f97-f52c-4686-bc49-737f7a6a8bbb}"), "Provider", "Provider name of the event"),
             new UIHints { Width = 240 });
 
+        private static readonly ColumnConfiguration TrackNameIdColumn = new ColumnConfiguration(
+            new ColumnMetadata(new Guid("{111094F9-BEB4-486F-AD60-3F53CFF702EA}"), "TrackNameId", "Track Name (Id)"),
+            new UIHints { Width = 240 });
+
+        private static readonly ColumnConfiguration ParentIdColumn = new ColumnConfiguration(
+            new ColumnMetadata(new Guid("{A77736C3-AC5C-4100-B246-3821A2E73B15}"), "Parent Id", "Parent Id"),
+            new UIHints { Width = 240 });
+
+        private static readonly ColumnConfiguration ParentDepthLevelColumn = new ColumnConfiguration(
+            new ColumnMetadata(new Guid("{89572E6E-86D5-4CBA-B0D4-4F9D2147BF50}"), "Parent Depth Level", "Parent Depth Level (0 is at Top of Tree)"),
+            new UIHints
+            {
+                Width = 70,
+                SortPriority = 0,
+                SortOrder = SortOrder.Ascending,
+            });
+
         // Need 2 of these with different sorting
         const string CountColumnGuid = "{99192cbf-5888-4873-a3b3-4faf5beaea15}";
         private static readonly ColumnConfiguration CountColumn = new ColumnConfiguration(
@@ -161,6 +178,23 @@ public static void BuildTable(ITableBuilder tableBuilder, IDataExtensionRetrieva
                 genericEventProjection.Compose((genericEvent) => genericEvent.Provider));
             tableGenerator.AddColumn(providerColumn);
 
+            var trackNameIdColumn = new BaseDataColumn<string>(
+                TrackNameIdColumn,
+                genericEventProjection.Compose((genericEvent) => genericEvent.ThreadTrack != null ? 
+                                                                    (!String.IsNullOrWhiteSpace(genericEvent.ThreadTrack.Name) ? $"{genericEvent.ThreadTrack.Name} ({genericEvent.ThreadTrack.Id})" : genericEvent.ThreadTrack.Id.ToString()) 
+                                                                    : String.Empty));
+            tableGenerator.AddColumn(trackNameIdColumn);
+
+            var parentIdColumn = new BaseDataColumn<long>(
+                ParentIdColumn,
+                genericEventProjection.Compose((genericEvent) => genericEvent.ParentId.HasValue? genericEvent.ParentId.Value : -1));
+            tableGenerator.AddColumn(parentIdColumn);
+
+            var parentDepthLevelColumn = new BaseDataColumn<int>(
+                ParentDepthLevelColumn,
+                genericEventProjection.Compose((genericEvent) => genericEvent.ParentTreeDepthLevel));
+            tableGenerator.AddColumn(parentDepthLevelColumn);
+
             tableGenerator.AddColumn(CountColumn, Projection.Constant<int>(1));
 
             // The provider column is optionally populated depending on whether or not the user specified a ProviderGUID mapping file
@@ -240,8 +274,9 @@ public static void BuildTable(ITableBuilder tableBuilder, IDataExtensionRetrieva
             SetGraphTableConfig(processThreadActivityConfig);
 
             var processThreadNameColumns = new List<ColumnConfiguration>(defaultColumns);
+            processThreadNameColumns.Insert(3, ParentDepthLevelColumn);
             processThreadNameColumns.Remove(EventNameColumn);
-            processThreadNameColumns.Insert(3, EventNameColumn);
+            processThreadNameColumns.Insert(4, EventNameColumn);
             var processThreadNameConfig = new TableConfiguration("Perfetto Trace Events - Process-Thread-Name")
             {
                 Columns = processThreadNameColumns,

PerfettoProcessor/Events/PerfettoArgEvent.cs

@@ -13,9 +13,9 @@ public class PerfettoArgEvent : PerfettoSqlEvent
         public long ArgSetId { get; set; }
         public string Flatkey { get; set; }
         public string ArgKey { get; set; }
-        public long IntValue { get; set; }
+        public long? IntValue { get; set; }
         public string StringValue { get; set; }
-        public double RealValue { get; set; }
+        public double? RealValue { get; set; }
         public string ValueType { get; set; }
 
         public override string GetSqlQuery()

PerfettoProcessor/Events/PerfettoMetadataEvent.cs

@@ -17,7 +17,7 @@ public class PerfettoMetadataEvent : PerfettoSqlEvent
         public string Name { get; set; }
         public string KeyType { get; set; }
 
-        public long IntValue { get; set; }
+        public long? IntValue { get; set; }
         public string StrValue { get; set; }
 
 

PerfettoProcessor/Events/PerfettoProcessEvent.cs

@@ -15,13 +15,13 @@ public class PerfettoProcessEvent : PerfettoSqlEvent
         public string Type { get; set; }
         public long Pid { get; set; }
         public string Name { get; set; }
-        public long StartTimestamp { get; set; }
-        public long RelativeStartTimestamp { get; set; }
-        public long EndTimestamp{ get; set; }
-        public long RelativeEndTimestamp { get; set; }
-        public long ParentUpid { get; set; }
-        public long Uid { get; set; }
-        public long AndroidAppId { get; set; }
+        public long? StartTimestamp { get; set; }
+        public long? RelativeStartTimestamp { get; set; }
+        public long? EndTimestamp{ get; set; }
+        public long? RelativeEndTimestamp { get; set; }
+        public long? ParentUpid { get; set; }
+        public long? Uid { get; set; }
+        public long? AndroidAppId { get; set; }
         public string CmdLine { get; set; }
         public long ArgSetId { get; set; }
 

PerfettoProcessor/Events/PerfettoProcessTrackEvent.cs

@@ -14,7 +14,7 @@ public class PerfettoProcessTrackEvent : PerfettoSqlEvent
         public long Id { get; set; }
         public string Type { get; set; }
         public string Name { get; set; }
-        public long SourceArgSetId { get; set; }
+        public long? SourceArgSetId { get; set; }
         public long Upid { get; set; }
 
         public override string GetSqlQuery()

PerfettoProcessor/Events/PerfettoSliceEvent.cs

@@ -9,7 +9,8 @@ public class PerfettoSliceEvent : PerfettoSqlEvent
     {
         public const string Key = "PerfettoSliceEvent";
 
-        public static string SqlQuery = "select ts, dur, arg_set_id, track_id, name, type, category from slice order by ts";
+        public static string SqlQuery = "select id, ts, dur, arg_set_id, track_id, name, type, category, parent_id from slice order by id";
+        public long Id { get; set; }
         public string Name { get; set; }
         public string Type { get; set; }
         public long Duration { get; set; }
@@ -18,6 +19,7 @@ public class PerfettoSliceEvent : PerfettoSqlEvent
         public long RelativeTimestamp { get; set; }
         public string Category { get; set; }
         public long TrackId { get; set; }
+        public long? ParentId { get; set; }
 
         public override string GetSqlQuery()
         {
@@ -45,6 +47,9 @@ public override void ProcessCell(string colName,
                     var longVal = batch.VarintCells[counters.IntCounter++];
                     switch (col)
                     {
+                        case "id":
+                            Id = longVal;
+                            break;
                         case "ts":
                             Timestamp = longVal;
                             break;
@@ -57,6 +62,9 @@ public override void ProcessCell(string colName,
                         case "track_id":
                             TrackId = longVal;
                             break;
+                        case "parent_id":
+                            ParentId = longVal;
+                            break;
                     }
 
                     break;