microsoft · KyleStorck · Sep 2, 2021 · Aug 31, 2021 · Aug 31, 2021 · Sep 1, 2021
diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoCpuCountersEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoCpuCountersEventCooker.cs
@@ -8,9 +8,10 @@
 using Microsoft.Performance.SDK.Extensibility.DataCooking;
 using Microsoft.Performance.SDK.Processing;
 using PerfettoCds.Pipeline.DataOutput;
+using PerfettoCds.Pipeline.SourceDataCookers;
 using PerfettoProcessor;
 
-namespace PerfettoCds.Pipeline.DataCookers
+namespace PerfettoCds.Pipeline.CompositeDataCookers
 {
     /// <summary>
     /// Pulls data from multiple individual SQL tables and joins them to create a CPU counters event.

diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoCpuFrequencyEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoCpuFrequencyEventCooker.cs
@@ -8,9 +8,10 @@
 using Microsoft.Performance.SDK.Extensibility.DataCooking;
 using Microsoft.Performance.SDK.Processing;
 using PerfettoCds.Pipeline.DataOutput;
+using PerfettoCds.Pipeline.SourceDataCookers;
 using PerfettoProcessor;
 
-namespace PerfettoCds.Pipeline.DataCookers
+namespace PerfettoCds.Pipeline.CompositeDataCookers
 {
     /// <summary>
     /// Pulls data from multiple individual SQL tables and joins them to create a a CPU frequency event. CPU frequency events

diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoCpuSchedEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoCpuSchedEventCooker.cs
@@ -8,9 +8,10 @@
 using Microsoft.Performance.SDK.Extensibility.DataCooking;
 using Microsoft.Performance.SDK.Processing;
 using PerfettoCds.Pipeline.DataOutput;
+using PerfettoCds.Pipeline.SourceDataCookers;
 using PerfettoProcessor;
 
-namespace PerfettoCds.Pipeline.DataCookers
+namespace PerfettoCds.Pipeline.CompositeDataCookers
 {
     /// <summary>
     /// Pulls data from multiple individual SQL tables and joins them to create a CPU scheduling event

diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoFtraceEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoFtraceEventCooker.cs
@@ -8,9 +8,10 @@
 using Microsoft.Performance.SDK.Extensibility.DataCooking;
 using Microsoft.Performance.SDK.Processing;
 using PerfettoCds.Pipeline.DataOutput;
+using PerfettoCds.Pipeline.SourceDataCookers;
 using PerfettoProcessor;
 
-namespace PerfettoCds.Pipeline.DataCookers
+namespace PerfettoCds.Pipeline.CompositeDataCookers
 {
     /// <summary>
     /// Pulls data from multiple individual SQL tables and joins them to create an Ftrace Perfetto event

diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoGenericEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoGenericEventCooker.cs
@@ -13,8 +13,9 @@
 using System.Xml;
 using System.Xml.Serialization;
 using System.IO;
+using PerfettoCds.Pipeline.SourceDataCookers;
 
-namespace PerfettoCds.Pipeline.DataCookers
+namespace PerfettoCds.Pipeline.CompositeDataCookers
 {
     /// <summary>
     /// XML deserialized EventProvider

diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoLogcatEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoLogcatEventCooker.cs
@@ -8,9 +8,10 @@
 using Microsoft.Performance.SDK.Extensibility.DataCooking;
 using Microsoft.Performance.SDK.Processing;
 using PerfettoCds.Pipeline.DataOutput;
+using PerfettoCds.Pipeline.SourceDataCookers;
 using PerfettoProcessor;
 
-namespace PerfettoCds.Pipeline.DataCookers
+namespace PerfettoCds.Pipeline.CompositeDataCookers
 {
     /// <summary>
     /// Pulls data from multiple individual SQL tables and joins them to create events for logcat output

diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoProcessMemoryEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoProcessMemoryEventCooker.cs
@@ -0,0 +1,133 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Performance.SDK;
+using Microsoft.Performance.SDK.Extensibility;
+using Microsoft.Performance.SDK.Extensibility.DataCooking;
+using Microsoft.Performance.SDK.Processing;
+using PerfettoCds.Pipeline.DataOutput;
+using PerfettoCds.Pipeline.SourceDataCookers;
+using PerfettoProcessor;
+
+namespace PerfettoCds.Pipeline.CompositeDataCookers
+{
+    /// <summary>
+    /// Pulls data from multiple individual SQL tables and joins them to create a process memory event. Process
+    /// memory events list different memory counts per process
+    /// </summary>
+    public sealed class PerfettoProcessMemoryEventCooker : CookedDataReflector, ICompositeDataCookerDescriptor
+    {
+        public static readonly DataCookerPath DataCookerPath = PerfettoPluginConstants.ProcessMemoryEventCookerPath;
+
+        public string Description => "Process memory composite cooker";
+
+        public DataCookerPath Path => DataCookerPath;
+
+        // Declare all of the cookers that are used by this CompositeCooker.
+        public IReadOnlyCollection<DataCookerPath> RequiredDataCookers => new[]
+        {
+            PerfettoPluginConstants.CounterCookerPath,
+            PerfettoPluginConstants.ProcessCounterTrackCookerPath,
+            PerfettoPluginConstants.ProcessCookerPath
+        };
+
+        [DataOutput]
+        public ProcessedEventData<PerfettoProcessMemoryEvent> ProcessMemoryEvents { get; }
+
+        public PerfettoProcessMemoryEventCooker() : base(PerfettoPluginConstants.ProcessMemoryEventCookerPath)
+        { 
+            this.ProcessMemoryEvents =
+                new ProcessedEventData<PerfettoProcessMemoryEvent>();
+        }
+
+        public void OnDataAvailable(IDataExtensionRetrieval requiredData)
+        {
+            // Gather the data from all the SQL tables
+            var counterData = requiredData.QueryOutput<ProcessedEventData<PerfettoCounterEvent>>(new DataOutputPath(PerfettoPluginConstants.CounterCookerPath, nameof(PerfettoCounterCooker.CounterEvents)));
+            var processCounterTrackData = requiredData.QueryOutput<ProcessedEventData<PerfettoProcessCounterTrackEvent>>(new DataOutputPath(PerfettoPluginConstants.ProcessCounterTrackCookerPath, nameof(PerfettoProcessCounterTrackCooker.ProcessCounterTrackEvents)));
+            var processData = requiredData.QueryOutput<ProcessedEventData<PerfettoProcessEvent>>(new DataOutputPath(PerfettoPluginConstants.ProcessCookerPath, nameof(PerfettoProcessCooker.ProcessEvents)));
+
+            // Join them all together
+            // Counter table contains the memory count value, timestamp
+            // ProcessCounterTrack contains the UPID and memory type name. All the memory types we care about start with "mem."
+            // Process contains the process name
+            var joined = from counter in counterData
+                         join processCounterTrack in processCounterTrackData on counter.TrackId equals processCounterTrack.Id
+                         join process in processData on processCounterTrack.Upid equals process.Upid
+                         where processCounterTrack.Name.StartsWith("mem.")
+                         orderby counter.Timestamp ascending
+                         select new { counter, processCounterTrack, process };
+
+            // Create events out of the joined results
+            foreach (var processGroup in joined.GroupBy(x => x.processCounterTrack.Upid))
+            {
+                var timeGroups = processGroup.GroupBy(z => z.counter.RelativeTimestamp);
+
+                for (int i = 0; i < timeGroups.Count(); i++)
+                {
+                    var timeGroup = timeGroups.ElementAt(i);
+
+                    var ts = timeGroup.Key;
+                    var processName = $"{timeGroup.ElementAt(0).process.Name} {processGroup.Key}";
+                    long nextTs = ts;
+                    if (i < timeGroups.Count() - 1)
+                    {
+                        // Need to look ahead in the future at the next event to get the timestamp so that we can calculate the duration
+                        nextTs = timeGroups.ElementAt(i + 1).Key;
+                    }
+
+                    double virt = 0.0, rss = 0.0, rssAnon = 0.0, rssFile = 0.0, rssShMem = 0.0, rssHwm = 0.0, swap = 0.0, locked = 0.0;
+                    // Gather each type of memory
+                    foreach (var thing in timeGroup)
+                    {
+                        switch (thing.processCounterTrack.Name)
+                        {
+                            case "mem.virt":
+                                virt = thing.counter.FloatValue;
+                                break;
+                            case "mem.rss":
+                                rss = thing.counter.FloatValue;
+                                break;
+                            case "mem.rss.anon":
+                                rssAnon = thing.counter.FloatValue;
+                                break;
+                            case "mem.rss.file":
+                                rssFile = thing.counter.FloatValue;
+                                break;
+                            case "mem.rss.shmem":
+                                rssShMem = thing.counter.FloatValue;
+                                break;
+                            case "mem.rss.watermark":
+                                rssHwm = thing.counter.FloatValue;
+                                break;
+                            case "mem.locked":
+                                locked = thing.counter.FloatValue;
+                                break;
+                            case "mem.swap":
+                                swap = thing.counter.FloatValue;
+                                break;
+                        }
+                    }
+                    PerfettoProcessMemoryEvent ev = new PerfettoProcessMemoryEvent
+                    (
+                        processName,
+                        new Timestamp(ts),
+                        new TimestampDelta(nextTs - ts),
+                        rssAnon,
+                        rssShMem,
+                        rssFile,
+                        rssHwm,
+                        rss,
+                        locked,
+                        swap,
+                        virt
+                    );
+                    this.ProcessMemoryEvents.AddEvent(ev);
+                }
+            }
+            this.ProcessMemoryEvents.FinalizeData();
+        }
+    }
+}
diff --git a/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoSystemMemoryEventCooker.cs b/PerfettoCds/Pipeline/CompositeDataCookers/PerfettoSystemMemoryEventCooker.cs
@@ -0,0 +1,132 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Performance.SDK;
+using Microsoft.Performance.SDK.Extensibility;
+using Microsoft.Performance.SDK.Extensibility.DataCooking;
+using Microsoft.Performance.SDK.Processing;
+using PerfettoCds.Pipeline.DataOutput;
+using PerfettoCds.Pipeline.SourceDataCookers;
+using PerfettoProcessor;
+
+namespace PerfettoCds.Pipeline.CompositeDataCookers
+{
+    /// <summary>
+    /// Pulls data from multiple individual SQL tables and joins them to create a system memory event. System
+    /// memory events capture periodic system memory counts
+    /// </summary>
+    public sealed class PerfettoSystemMemoryEventCooker : CookedDataReflector, ICompositeDataCookerDescriptor
+    {
+        public static readonly DataCookerPath DataCookerPath = PerfettoPluginConstants.SystemMemoryEventCookerPath;
+
+        public string Description => "System memory composite cooker";
+
+        public DataCookerPath Path => DataCookerPath;
+
+        // Declare all of the cookers that are used by this CompositeCooker.
+        public IReadOnlyCollection<DataCookerPath> RequiredDataCookers => new[]
+        {
+            PerfettoPluginConstants.CounterCookerPath,
+            PerfettoPluginConstants.CounterTrackCookerPath
+        };
+
+        [DataOutput]
+        public ProcessedEventData<PerfettoSystemMemoryEvent> SystemMemoryEvents { get; }
+
+        // Perfetto captures memory counts from /proc/meminfo and outputs events with
+        // the following names.
+        // Set sys_stats_counters.h in Perfetto repo.
+        public HashSet<string> MemoryTypes = new HashSet<string>() {
+            "MemUnspecified",
+            "MemTotal",
+            "MemFree",
+            "MemAvailable",
+            "Buffers",
+            "Cached",
+            "SwapCached",
+            "Active",
+            "Inactive",
+            "Active(anon)",
+            "Inactive(anon)",
+            "Active(file)",
+            "Inactive(file)",
+            "Unevictable",
+            "Mlocked",
+            "SwapTotal",
+            "SwapFree",
+            "Dirty",
+            "Writeback",
+            "AnonPages",
+            "Mapped",
+            "Shmem",
+            "Slab",
+            "SReclaimable",
+            "SUnreclaim",
+            "KernelStack",
+            "PageTables",
+            "CommitLimit",
+            "Committed_AS",
+            "VmallocTotal",
+            "VmallocUsed",
+            "VmallocChunk",
+            "CmaTotal",
+            "CmaFree"
+        };
+
+        public PerfettoSystemMemoryEventCooker() : base(PerfettoPluginConstants.SystemMemoryEventCookerPath)
+        { 
+            this.SystemMemoryEvents =
+                new ProcessedEventData<PerfettoSystemMemoryEvent>();
+        }
+
+        public void OnDataAvailable(IDataExtensionRetrieval requiredData)
+        {
+            // Gather the data from all the SQL tables
+            var counterData = requiredData.QueryOutput<ProcessedEventData<PerfettoCounterEvent>>(new DataOutputPath(PerfettoPluginConstants.CounterCookerPath, nameof(PerfettoCounterCooker.CounterEvents)));
+            var counterTrackData = requiredData.QueryOutput<ProcessedEventData<PerfettoCounterTrackEvent>>(new DataOutputPath(PerfettoPluginConstants.CounterTrackCookerPath, nameof(PerfettoCounterTrackCooker.CounterTrackEvents)));
+
+            // Join them all together
+            // Counter table contains the memory count value, timestamp
+            // counterTrackData contains the name of the memory type
+            // Process contains the process name
+            var joined = from counter in counterData
+                         join counterTrack in counterTrackData on counter.TrackId equals counterTrack.Id
+                         where MemoryTypes.Contains(counterTrack.Name)
+                         orderby counter.Timestamp ascending
+                         select new { counter, counterTrack };
+
+            // Create events out of the joined results
+            foreach (var memoryGroup in joined.GroupBy(x => x.counterTrack.Name))
+            {
+                string memoryType = memoryGroup.Key;
+
+                for(int i = 0; i < memoryGroup.Count(); i++)
+                {
+                    var thing = memoryGroup.ElementAt(i);
+                    double val = thing.counter.FloatValue;
+                    var ts = thing.counter.RelativeTimestamp;
+
+                    long nextTs = ts;
+                    if (i < memoryGroup.Count() - 1)
+                    {
+                        // Need to look ahead in the future at the next event to get the timestamp so that we can calculate the duration
+                        nextTs = memoryGroup.ElementAt(i + 1).counter.RelativeTimestamp;
+                    }
+
+                    PerfettoSystemMemoryEvent ev = new PerfettoSystemMemoryEvent
+                    (
+                        val, 
+                        memoryType,
+                        new Timestamp(ts),
+                        new TimestampDelta(nextTs - ts)
+                    );
+                    this.SystemMemoryEvents.AddEvent(ev);
+                }
+
+            }
+            this.SystemMemoryEvents.FinalizeData();
+        }
+    }
+}
diff --git a/PerfettoCds/Pipeline/DataOutput/PerfettoProcessMemoryEvent.cs b/PerfettoCds/Pipeline/DataOutput/PerfettoProcessMemoryEvent.cs
@@ -0,0 +1,58 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+using Microsoft.Performance.SDK;
+using System.Collections.Generic;
+
+namespace PerfettoCds.Pipeline.DataOutput
+{
+    /// <summary>
+    /// A event that represents several memory values for a process at a point in time
+    /// </summary>
+    public readonly struct PerfettoProcessMemoryEvent
+    {
+        public string ProcessName { get; }
+        public Timestamp StartTimestamp { get; }
+        public TimestampDelta Duration { get; }
+
+        /// Resident set size - anonymous memory
+        public double RssAnon { get; }
+        /// Resident set size - shared memory
+        public double RssShMem { get; }
+        /// Resident set size - file mappings
+        public double RssFile { get; }
+        /// Resident set size - Peak (high water mark)
+        public double RssHwm { get; }
+        /// Resident set size - Sum of anon, file, ShMem
+        public double Rss { get; }
+        /// Locked memory size
+        public double Locked { get; }
+        /// Swapped out VM size by anonymous private pages
+        public double Swap { get; }
+        /// Peak virtual memory size
+        public double Virt { get; }
+
+        public PerfettoProcessMemoryEvent(string processName, Timestamp startTimestamp, TimestampDelta duration,
+            double rssAnon,
+            double rssShMem,
+            double rssFile,
+            double rssHwm,
+            double rss,
+            double locked,
+            double swap,
+            double virt)
+        {
+            this.ProcessName = processName;
+            this.StartTimestamp = startTimestamp;
+            this.Duration = duration;
+
+            this.RssAnon = rssAnon;
+            this.Locked = locked;
+            this.RssShMem = rssShMem;
+            this.RssFile = rssFile;
+            this.RssHwm = rssHwm;
+            this.Rss = rss;
+            this.Swap = swap;
+            this.Virt = virt;
+        }
+    }
+}