[smuxi: 140/179] Engine-MessageBuffer: support multiple source files for cat and copy actions

[Mirco M. M. Bauer <mmmbauer src gnome org>]

commit dcadd8cef03a705800771433671d1d2232fc466c
Author: Mirco Bauer <meebey meebey net>
Date:   Tue May 2 18:33:31 2017 +0800

    Engine-MessageBuffer: support multiple source files for cat and copy actions
    
    If someone wants to migrate existing message buffer files from multiple machines
    that were running local Smuxi engines to a single smuxi-server one, it can be
    useful to merge multiple message buffer files into a new one. The easiest way to
    achieve this is by allowing to read from multiple buffer files in the same run
    and simply concatenate all messages to the JSON pipe or a new destination
    message buffer.
    
    The use of IEnumerable<T>.Concat() can be pretty harmful but not in this
    particular case as the stack height / recursion depth is only as height as the
    number of message buffers were passed from the command line. For more details
    why this can be dangerous, [see here][harmful-concat].
    
     [harmful-concat]: 
http://programmaticallyspeaking.com/how-enumerableconcat-brought-down-a-production-server.html

 src/Engine-MessageBuffer/Main.cs                |   86 ++++++++++++++++-------
 src/Engine-MessageBuffer/smuxi-message-buffer.1 |    4 +-
 2 files changed, 62 insertions(+), 28 deletions(-)
---
diff --git a/src/Engine-MessageBuffer/Main.cs b/src/Engine-MessageBuffer/Main.cs
index 8237542..40263af 100644
--- a/src/Engine-MessageBuffer/Main.cs
+++ b/src/Engine-MessageBuffer/Main.cs
@@ -147,7 +147,7 @@ namespace Smuxi.Engine
                         )
                     );
                     Console.WriteLine();
-                    Console.WriteLine("  db_path " + _("Database path"));
+                    Console.WriteLine("  db_path(s)... " + _("Database path(s)"));
                     Console.WriteLine();
                     Console.WriteLine(_("Options:"));
                     parser.WriteOptionDescriptions(Console.Out);
@@ -162,8 +162,8 @@ namespace Smuxi.Engine
                     action
                 );
             }
-            var dbPath = parameters[0];
-            Copy(dbPath, dbFormat, null, null);
+            var dbPaths = parameters.ToArray();
+            Copy(dbPaths, dbFormat, null, null);
         }
 
         static void CopyAction(string action, IEnumerable<string> args)
@@ -206,12 +206,12 @@ namespace Smuxi.Engine
                 val => {
                     Console.WriteLine(
                         String.Format(
-                            _("Usage: smuxi-message-buffer {0} [action-options] source_db destination_db"),
+                            _("Usage: smuxi-message-buffer {0} [action-options] source_db(s)... 
destination_db"),
                             action
                         )
                     );
                     Console.WriteLine();
-                    Console.WriteLine("  source_db " + _("Source file path"));
+                    Console.WriteLine("  source_db(s)... " + _("Source file path(s)"));
                     Console.WriteLine("  destination_db " + _("Destination file path or -/empty for 
stdout"));
                     Console.WriteLine();
                     Console.WriteLine(_("Options:"));
@@ -227,12 +227,12 @@ namespace Smuxi.Engine
                     action
                 );
             }
-            var sourceFile = parameters[0];
-            var destinationFile = parameters[1];
+            var sourceFiles = parameters.Take(parameters.Count - 1).ToArray();
+            var destinationFile = parameters.Last();
             if (destinationFile == "-") {
                 destinationFile = "";
             }
-            Copy(sourceFile, sourceFormat, destinationFile, destinationFormat);
+            Copy(sourceFiles, sourceFormat, destinationFile, destinationFormat);
         }
 
         static void Copy(string sourceFile, string sourceFormat,
@@ -241,11 +241,24 @@ namespace Smuxi.Engine
             if (String.IsNullOrEmpty(sourceFile)) {
                 throw new ArgumentException(_("sourceFile must not be empty."));
             }
+            Copy(new string[] { sourceFile }, sourceFormat, destinationFile, destinationFormat);
+        }
+
+        static void Copy(string[] sourceFiles, string sourceFormat,
+                         string destinationFile, string destinationFormat)
+        {
+            if (sourceFiles == null || sourceFiles.Length == 0) {
+                throw new ArgumentException(_("sourceFiles must not be empty."));
+            }
 
-            IMessageBuffer sourceBuffer = null, destinationBuffer = null;
+            var sourceBuffers = new List<IMessageBuffer>();
+            IMessageBuffer destinationBuffer = null;
             try {
-                var sourceBufferType = ParseMessageBufferType(sourceFile, sourceFormat);
-                sourceBuffer = CreateMessageBuffer(sourceFile, sourceBufferType);
+                foreach (var sourceFile in sourceFiles) {
+                    var sourceBufferType = ParseMessageBufferType(sourceFile, sourceFormat);
+                    var sourceBuffer = CreateMessageBuffer(sourceFile, sourceBufferType);
+                    sourceBuffers.Add(sourceBuffer);
+                }
 
                 if (!String.IsNullOrEmpty(destinationFile)) {
                     var destinationBufferType = ParseMessageBufferType(destinationFile,
@@ -262,29 +275,23 @@ namespace Smuxi.Engine
                     }
                 }
 
+                // append all messages of all source buffers together in a lazy way
+                IEnumerable<MessageModel> concatenatedMessages = new List<MessageModel>(0);
+                sourceBuffers.ForEach(x =>
+                    concatenatedMessages = concatenatedMessages.Concat(x)
+                );
+
                 if (destinationBuffer == null) {
                     // JSON pipe
-                    Console.WriteLine("[");
-                    var msgCount = sourceBuffer.Count;
-                    var i = 0;
-                    foreach (var msg in sourceBuffer) {
-                        var dto = new MessageDtoModelV1(msg);
-                        var json = JsonSerializer.SerializeToString(dto);
-                        if (i++ < msgCount - 1) {
-                            Console.WriteLine("{0},", json);
-                        } else {
-                            Console.WriteLine(json);
-                        }
-                    }
-                    Console.WriteLine("]");
+                    WriteMessagesToJson(concatenatedMessages, Console.Out);
                 } else {
-                    foreach (var msg in sourceBuffer) {
+                    foreach (var msg in concatenatedMessages) {
                         destinationBuffer.Add(msg);
                     }
                     destinationBuffer.Flush();
                 }
             } finally {
-                if (sourceBuffer != null) {
+                foreach (var sourceBuffer in sourceBuffers) {
                     sourceBuffer.Dispose();
                 }
                 if (destinationBuffer != null) {
@@ -293,6 +300,33 @@ namespace Smuxi.Engine
             }
         }
 
+        static void WriteMessagesToJson(IEnumerable<MessageModel> messages, TextWriter writer)
+        {
+            // OPT: if you are wondering why this code is handling the
+            // serialization of JSON list manually instead of passing it as a
+            // List<T> in a single method call to JsonSerializer.SerializeToWriter(dtoMessages)
+            // then this is because it would mean that all messages from the
+            // source message buffer would need to be read completely into
+            // memory before serializing it into JSON and then writing the result
+            // of that to the console or file. Instead this is a read one message
+            // from the message buffer, copy it to a DTO object, serialize that
+            // one message to JSON and then write that single JSON object to the
+            // target which is a TextWriter.
+            writer.Write("[");
+            bool first = true;
+            foreach (var message in messages) {
+                if (first) {
+                    first = false;
+                } else {
+                    writer.Write(",");
+                }
+                var dtoMessage = new MessageDtoModelV1(message);
+                JsonSerializer.SerializeToWriter(dtoMessage, writer);
+            }
+            writer.WriteLine("]");
+            writer.Flush();
+        }
+
         static MessageBufferType ParseMessageBufferType(string fileName, string type)
         {
             if (String.IsNullOrEmpty(type)) {
diff --git a/src/Engine-MessageBuffer/smuxi-message-buffer.1 b/src/Engine-MessageBuffer/smuxi-message-buffer.1
index d864818..33100ba 100644
--- a/src/Engine-MessageBuffer/smuxi-message-buffer.1
+++ b/src/Engine-MessageBuffer/smuxi-message-buffer.1
@@ -10,13 +10,13 @@
 convert
 .Op Fl \-source-format Ns = Ns Ar format
 .Op Fl \-destination-format Ns = Ns Ar format
-.Ar source-file
+.Ar source-file(s)...
 .Ar destination-file
 .Nm smuxi-message-buffer
 .Op Fl dh
 cat
 .Op Fl \-format Ns = Ns Ar format
-.Ar source-file
+.Ar source-file(s)...
 .Sh DESCRIPTION
 .Nm
 is a tool that manages the databases that Smuxi holds chat histories in. Currently, it can convert and dump 
databases.