[banshee] [extras/metrics] Better analysis
- From: Gabriel Burt <gburt src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [banshee] [extras/metrics] Better analysis
- Date: Sun, 28 Feb 2010 00:30:15 +0000 (UTC)
commit 424a345b1cbde80b9bfb15b32580d224df576507
Author: Gabriel Burt <gabriel burt gmail com>
Date: Sat Feb 27 16:29:19 2010 -0800
[extras/metrics] Better analysis
Use SqliteModelCache to select the most recent sample for each user
for each metric.
extras/metrics/Database.cs | 16 +++-
extras/metrics/MetaMetrics.cs | 177 ++++++++++++++++++++++--------------
extras/metrics/MultiUserSample.cs | 22 +++--
3 files changed, 133 insertions(+), 82 deletions(-)
---
diff --git a/extras/metrics/Database.cs b/extras/metrics/Database.cs
index f325702..8bad4ad 100644
--- a/extras/metrics/Database.cs
+++ b/extras/metrics/Database.cs
@@ -41,20 +41,25 @@ namespace metrics
public static HyenaSqliteConnection Open ()
{
+ HyenaSqliteCommand.LogAll = ApplicationContext.CommandLine.Contains ("debug-sql");
var db = new HyenaSqliteConnection (db_path);
db.Execute ("PRAGMA cache_size = ?", 32768 * 2);
db.Execute ("PRAGMA synchronous = OFF");
db.Execute ("PRAGMA temp_store = MEMORY");
db.Execute ("PRAGMA count_changes = OFF");
+ SampleProvider = new SqliteModelProvider<MultiUserSample> (db, "Samples", true);
return db;
}
public static bool Exists { get { return System.IO.File.Exists (db_path); } }
+ public static SqliteModelProvider<MultiUserSample> SampleProvider { get; private set; }
+
public static void Import ()
{
using (var db = Open ()) {
- MultiUserSample.Provider = new SqliteModelProvider<MultiUserSample> (db, "Samples", true);
+ var sample_provider = SampleProvider;
+ db.BeginTransaction ();
foreach (var file in System.IO.Directory.GetFiles ("data")) {
Log.InformationFormat ("Importing {0}", file);
@@ -69,23 +74,21 @@ namespace metrics
}
var metrics = o["Metrics"] as JsonObject;
- db.BeginTransaction ();
try {
foreach (string metric_name in metrics.Keys) {
var samples = metrics[metric_name] as JsonArray;
foreach (JsonArray sample in samples) {
- MultiUserSample.Import (user_id, metric_name, (string)sample[0], (object)sample[1]);
+ sample_provider.Save (MultiUserSample.Import (user_id, metric_name, (string)sample[0], (object)sample[1]));
}
}
- db.CommitTransaction ();
} catch {
- db.RollbackTransaction ();
throw;
}
} catch (Exception e) {
Log.Exception (String.Format ("Failed to read {0}", file), e);
}
}
+ db.CommitTransaction ();
}
}
}
@@ -123,6 +126,9 @@ namespace metrics
public override object Final (object contextData)
{
var list = contextData as List<T>;
+ if (list == null || list.Count == 0)
+ return null;
+
list.Sort ();
return list[list.Count / 2];
}
diff --git a/extras/metrics/MetaMetrics.cs b/extras/metrics/MetaMetrics.cs
index e4040a7..00ce9a3 100644
--- a/extras/metrics/MetaMetrics.cs
+++ b/extras/metrics/MetaMetrics.cs
@@ -30,51 +30,125 @@ using System.Collections;
namespace metrics
{
- public class MetaMetrics
+ public class SampleModel : ICacheableDatabaseModel
{
- private HyenaSqliteConnection db;
+ public string ReloadFragment { get; set; }
+ public string SelectAggregates { get; set; }
+ public string JoinTable { get; set; }
+ public string JoinFragment { get; set; }
+ public string JoinPrimaryKey { get; set; }
+ public string JoinColumn { get; set; }
+ public bool CachesJoinTableEntries { get; set; }
+ public bool CachesValues { get; set; }
+
+ public int FetchCount {
+ get { return 10; }
+ }
+ public Hyena.Collections.Selection Selection { get; private set; }
- public DateTime FirstReport { get; private set; }
- public DateTime LastReport { get; private set; }
+ public SqliteModelCache<MultiUserSample> Cache { get; private set; }
- public MetaMetrics (HyenaSqliteConnection db)
+ private static int id;
+
+ public SampleModel (string condition, HyenaSqliteConnection db, string aggregates)
{
- this.db = db;
- FirstReport = db.Query<DateTime> ("SELECT Stamp FROM Samples ORDER BY STAMP ASC");
- LastReport = db.Query<DateTime> ("SELECT Stamp FROM Samples ORDER BY STAMP DESC");
+ Selection = new Hyena.Collections.Selection ();
+ ReloadFragment = String.Format ("FROM Samples {0}", condition);
+ SelectAggregates = aggregates;
+ Cache = new SqliteModelCache<MultiUserSample> (db, (id++).ToString (), this, Database.SampleProvider);
+ }
- Console.WriteLine ("First report was on {0}", FirstReport);
- Console.WriteLine ("Last report was on {0}", LastReport);
- Console.WriteLine ("Total unique users: {0}", db.Query<long> ("SELECT COUNT(DISTINCT(UserId)) FROM Samples"));
+ public void Reload ()
+ {
+ Cache.Reload ();
+ Cache.UpdateAggregates ();
+ }
+ }
- var metrics = db.QueryEnumerable<string> ("SELECT DISTINCT(MetricName) as name FROM Samples ORDER BY name ASC");
+ public class MetricSampleModel : SampleModel
+ {
+ public string MetricName { get; private set; }
- foreach (var metric in metrics) {
- //var pieces = metric.Split ('/');
- //var name = pieces[pieces.Length - 1];
+ private string condition;
+ public MetricSampleModel (SqliteModelCache<MultiUserSample> limiter, HyenaSqliteConnection db, string aggregates) : base (null, db, aggregates)
+ {
+ condition = String.Format (
+ "FROM Samples, HyenaCache WHERE Samples.MetricName = '{0}' AND HyenaCache.ModelID = {1} AND Samples.ID = HyenaCache.ItemID",
+ "{0}", limiter.CacheId
+ );
+ }
+
+ public void ChangeMetric (string metricName)
+ {
+ MetricName = metricName;
+ ReloadFragment = String.Format (condition, metricName);
+ Reload ();
+ }
+ }
+
+ public class MetaMetrics
+ {
+ string fmt = "{0,20:N1}";
+ public MetaMetrics (HyenaSqliteConnection db)
+ {
+ var latest_samples = new SampleModel ("GROUP BY UserID, MetricName ORDER BY stamp desc", db, "COUNT(DISTINCT(UserID)), MIN(Stamp), MAX(Stamp)");
+ latest_samples.Cache.AggregatesUpdated += (reader) => {
+ Console.WriteLine ("Total unique users for this time slice: {0}", reader[1]);
+ Console.WriteLine ("First report was on {0}", SqliteUtils.FromDbFormat (typeof(DateTime), reader[2]));
+ Console.WriteLine ("Last report was on {0}", SqliteUtils.FromDbFormat (typeof(DateTime), reader[3]));
+ Console.WriteLine ();
+ };
+ latest_samples.Reload ();
+
+ var string_summary = new MetricSampleModel (latest_samples.Cache, db,
+ @"COUNT(DISTINCT(UserID))"
+ );
+ string_summary.Cache.AggregatesUpdated += (agg_reader) => {
+ Console.WriteLine (String.Format (" Users: {0}", fmt), agg_reader[1]);
+ using (var reader = new HyenaDataReader (db.Query (
+ @"SELECT COUNT(DISTINCT(UserId)) as users, Value FROM Samples, HyenaCache
+ WHERE MetricName = ? AND HyenaCache.ModelID = ? AND HyenaCache.ItemID = Samples.ID
+ GROUP BY Value ORDER BY users DESC", string_summary.MetricName, string_summary.Cache.CacheId))) {
+ while (reader.Read ()) {
+ Console.WriteLine (" {0,-5}: {1,-20}", reader.Get<long> (0), reader.Get<string> (1));
+ }
+ }
+ };
+
+ var numeric_slice = new MetricSampleModel (latest_samples.Cache, db,
+ @"MIN(CAST(Value as NUMERIC)), MAX(CAST(Value as NUMERIC)),
+ AVG(CAST(Value as NUMERIC)), HYENA_METRICS_MEDIAN_DOUBLE(CAST(Value as NUMERIC)), COUNT(DISTINCT(UserID))"
+ );
+
+ numeric_slice.Cache.AggregatesUpdated += (reader) => {
+ Console.WriteLine (String.Format (" Users: {0}", fmt), reader[5]);
+ Console.WriteLine (String.Format (" Min: {0}", fmt), reader[1]);
+ Console.WriteLine (String.Format (" Avg: {0}", fmt), reader[3]);
+ Console.WriteLine (String.Format (" Median: {0}", fmt), reader[4]);
+ Console.WriteLine (String.Format (" Max: {0}", fmt), reader[2]);
+ Console.WriteLine ();
+ };
+
+ var metrics = db.QueryEnumerable<string> ("SELECT DISTINCT(MetricName) as name FROM Samples ORDER BY name ASC");
+ foreach (var metric in metrics) {
switch (GetMetricType (metric)) {
- case "string": SummarizeTextual (metric); break;
- case "timespan" : SummarizeNumeric<TimeSpan> (metric); break;
- case "datetime" : SummarizeNumeric<DateTime> (metric); break;
- case "fixed": SummarizeNumeric<long> (metric); break;
- case "float": SummarizeNumeric<double> (metric); break;
+ case "string":
+ Console.WriteLine ("{0}:", metric);
+ string_summary.ChangeMetric (metric);
+ break;
+ //case "timespan" : SummarizeNumeric<TimeSpan> (metric); break;
+ //case "datetime" : SummarizeNumeric<DateTime> (metric); break;
+ case "float":
+ Console.WriteLine ("{0}:", metric);
+ //SummarizeNumeric<long> (metric_cache);
+ numeric_slice.ChangeMetric (metric);
+ break;
+ //case "float":
+ //SummarizeNumeric<double> (metric_cache);
+ //break;
}
}
-
- /*SummarizeTextual ("Env/CultureInfo");
- SummarizeTextual ("Banshee/Configuration/core/io_provider");
-
- SummarizeNumeric<TimeSpan> ("Banshee/RunDuration");
-
- SummarizeNumeric<long> ("Banshee/Screen/Width");
- SummarizeNumeric<long> ("Banshee/Screen/Height");
- SummarizeNumeric<long> ("Banshee/Screen/NMonitors");
- SummarizeTextual ("Banshee/Screen/IsComposited");*/
-
- /*foreach (var metric in db.QueryEnumerable<string> ("SELECT DISTINCT(MetricName) as name FROM Samples ORDER BY name ASC")) {
- Console.WriteLine (metric);
- }*/
}
private string GetMetricType (string name)
@@ -93,40 +167,5 @@ namespace metrics
return "string";
}
-
- private void SummarizeNumeric<T> (string metric_name)
- {
- Console.WriteLine ("{0}:", metric_name);
- var t = typeof(T);
- string fmt = t == typeof(DateTime) ? "{0}" : "{0,10:N1}";
- string median_func = "HYENA_METRICS_MEDIAN_" + (t == typeof(DateTime) ? "DATETIME" : t == typeof(long) ? "LONG" : "DOUBLE");
-
- using (var reader = new HyenaDataReader (db.Query (String.Format (@"
- SELECT
- MIN(CAST(Value as NUMERIC)), MAX(CAST(Value as NUMERIC)),
- AVG(CAST(Value as NUMERIC)), {0}(CAST(Value as NUMERIC))
- FROM Samples WHERE MetricName = ?", median_func), metric_name))) {
- if (reader.Read ()) {
- Console.WriteLine (String.Format (" Min: {0}", fmt), reader.Get<T> (0));
- Console.WriteLine (String.Format (" Avg: {0}", fmt), reader.Get<T> (2));
- Console.WriteLine (String.Format (" Median: {0}", fmt), reader.Get<T> (3));
- Console.WriteLine (String.Format (" Max: {0}", fmt), reader.Get<T> (1));
- }
- }
-
- Console.WriteLine ();
- }
-
- private void SummarizeTextual (string metric_name)
- {
- Console.WriteLine ("{0}:", metric_name);
- //Console.WriteLine (" Unique Values: {0,-20}", db.Query<long> ("SELECT COUNT(DISTINCT(Value)) FROM Samples WHERE MetricName = ?", metric_name));
- using (var reader = new HyenaDataReader (db.Query ("SELECT COUNT(DISTINCT(UserId)) as users, Value FROM Samples WHERE MetricName = ? GROUP BY Value ORDER BY users DESC", metric_name))) {
- while (reader.Read ()) {
- Console.WriteLine (" {0,-5}: {1,-20}", reader.Get<long> (0), reader.Get<string> (1));
- }
- }
- Console.WriteLine ();
- }
}
}
\ No newline at end of file
diff --git a/extras/metrics/MultiUserSample.cs b/extras/metrics/MultiUserSample.cs
index 0eac2c8..8fd2f0d 100644
--- a/extras/metrics/MultiUserSample.cs
+++ b/extras/metrics/MultiUserSample.cs
@@ -33,30 +33,36 @@ using Hyena.Json;
namespace metrics
{
- public class MultiUserSample : Sample
+ public class MultiUserSample : Sample, Hyena.Data.ICacheableItem
{
- public static SqliteModelProvider<MultiUserSample> Provider;
-
- [DatabaseColumn]
+ [DatabaseColumn (Index = "SampleUserIdIndex")]
public string UserId;
+ // ICacheableItem
+ public object CacheEntryId { get; set; }
+ public long CacheModelId { get; set; }
+
public MultiUserSample ()
{
}
static DateTime value_dt;
static TimeSpan value_span;
- public static void Import (string user_id, string metric_name, string stamp, object val)
+ public static MultiUserSample Import (string user_id, string metric_name, string stamp, object val)
{
var sample = new MultiUserSample ();
sample.UserId = user_id;
+
+ // TODO collapse various DAP and DAAP library stats?
sample.MetricName = metric_name;
DateTime stamp_dt;
- if (DateTimeUtil.TryParseInvariant (stamp, out stamp_dt)) {
- sample.Stamp = stamp_dt;
+ if (!DateTimeUtil.TryParseInvariant (stamp, out stamp_dt)) {
+ Hyena.Log.Error ("Invalid stamp: ", stamp);
+ return null;
}
+ sample.Stamp = stamp_dt;
string value_str = val as string;
if (value_str != null) {
@@ -72,7 +78,7 @@ namespace metrics
sample.SetValue (val);
}
- Provider.Save (sample);
+ return sample;
}
}
}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
