[hyena] Added new Hyena.Downloader namespace
- From: Gabriel Burt <gburt src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [hyena] Added new Hyena.Downloader namespace
- Date: Fri, 9 Jul 2010 21:31:43 +0000 (UTC)
commit 45a39d043bc36e77539b4366b96ec4df2ea85b81
Author: Aaron Bockover <abockover novell com>
Date: Wed Jul 7 18:17:54 2010 -0400
Added new Hyena.Downloader namespace
Provides a basic async HTTP downloader service.
Hyena/Hyena.Downloader/Buffer.cs | 38 ++++
Hyena/Hyena.Downloader/DownloadManager.cs | 140 ++++++++++++++
Hyena/Hyena.Downloader/HttpDownloader.cs | 256 +++++++++++++++++++++++++
Hyena/Hyena.Downloader/HttpDownloaderState.cs | 44 +++++
Hyena/Hyena.Downloader/HttpFileDownloader.cs | 85 ++++++++
Hyena/Hyena.csproj | 5 +
Hyena/Makefile.am | 5 +
7 files changed, 573 insertions(+), 0 deletions(-)
---
diff --git a/Hyena/Hyena.Downloader/Buffer.cs b/Hyena/Hyena.Downloader/Buffer.cs
new file mode 100644
index 0000000..97792b3
--- /dev/null
+++ b/Hyena/Hyena.Downloader/Buffer.cs
@@ -0,0 +1,38 @@
+//
+// Buffer.cs
+//
+// Author:
+// Aaron Bockover <abockover novell com>
+//
+// Copyright 2010 Novell, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+using System;
+
+namespace Hyena.Downloader
+{
+ public class Buffer
+ {
+ public long Length { get; set; }
+ public DateTime TimeStamp { get; set; }
+ public byte [] Data { get; set; }
+ }
+}
+
diff --git a/Hyena/Hyena.Downloader/DownloadManager.cs b/Hyena/Hyena.Downloader/DownloadManager.cs
new file mode 100644
index 0000000..6d12943
--- /dev/null
+++ b/Hyena/Hyena.Downloader/DownloadManager.cs
@@ -0,0 +1,140 @@
+//
+// DownloadManager.cs
+//
+// Author:
+// Aaron Bockover <abockover novell com>
+//
+// Copyright 2010 Novell, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+using System;
+using System.Collections.Generic;
+
+namespace Hyena.Downloader
+{
+ public class DownloadManager
+ {
+ private object sync_root = new object ();
+ protected object SyncRoot {
+ get { return sync_root; }
+ }
+
+ private Queue<HttpDownloader> pending_downloaders = new Queue<HttpDownloader> ();
+ private List<HttpDownloader> active_downloaders = new List<HttpDownloader> ();
+
+ protected Queue<HttpDownloader> PendingDownloaders {
+ get { return pending_downloaders; }
+ }
+
+ protected List<HttpDownloader> ActiveDownloaders {
+ get { return active_downloaders; }
+ }
+
+ public event Action<HttpDownloader> Started;
+ public event Action<HttpDownloader> Finished;
+ public event Action<HttpDownloader> Progress;
+ public event Action<HttpDownloader> BufferUpdated;
+
+ public int MaxConcurrentDownloaders { get; set; }
+
+ public int PendingDownloadCount {
+ get { return pending_downloaders.Count; }
+ }
+
+ public int ActiveDownloadCount {
+ get { return active_downloaders.Count; }
+ }
+
+ public int TotalDownloadCount {
+ get { return PendingDownloadCount + ActiveDownloadCount; }
+ }
+
+ public DownloadManager ()
+ {
+ MaxConcurrentDownloaders = 2;
+ }
+
+ public void QueueDownloader (HttpDownloader downloader)
+ {
+ lock (SyncRoot) {
+ pending_downloaders.Enqueue (downloader);
+ Update ();
+ }
+ }
+
+ public void WaitUntilFinished ()
+ {
+ while (TotalDownloadCount > 0);
+ }
+
+ private void Update ()
+ {
+ lock (SyncRoot) {
+ while (pending_downloaders.Count > 0 && active_downloaders.Count < MaxConcurrentDownloaders) {
+ var downloader = pending_downloaders.Peek ();
+ downloader.Started += OnDownloaderStarted;
+ downloader.Finished += OnDownloaderFinished;
+ downloader.Progress += OnDownloaderProgress;
+ downloader.BufferUpdated += OnDownloaderBufferUpdated;
+ active_downloaders.Add (downloader);
+ pending_downloaders.Dequeue ();
+ downloader.Start ();
+ }
+ }
+ }
+
+ protected virtual void OnDownloaderStarted (HttpDownloader downloader)
+ {
+ var handler = Started;
+ if (handler != null) {
+ handler (downloader);
+ }
+ }
+
+ protected virtual void OnDownloaderFinished (HttpDownloader downloader)
+ {
+ lock (SyncRoot) {
+ active_downloaders.Remove (downloader);
+ Update ();
+ }
+
+ var handler = Finished;
+ if (handler != null) {
+ handler (downloader);
+ }
+ }
+
+ protected virtual void OnDownloaderProgress (HttpDownloader downloader)
+ {
+ var handler = Progress;
+ if (handler != null) {
+ handler (downloader);
+ }
+ }
+
+ protected virtual void OnDownloaderBufferUpdated (HttpDownloader downloader)
+ {
+ var handler = BufferUpdated;
+ if (handler != null) {
+ handler (downloader);
+ }
+ }
+ }
+}
diff --git a/Hyena/Hyena.Downloader/HttpDownloader.cs b/Hyena/Hyena.Downloader/HttpDownloader.cs
new file mode 100644
index 0000000..72a2572
--- /dev/null
+++ b/Hyena/Hyena.Downloader/HttpDownloader.cs
@@ -0,0 +1,256 @@
+//
+// HttpDownloader.cs
+//
+// Author:
+// Aaron Bockover <abockover novell com>
+//
+// Copyright 2010 Novell, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+using System;
+using System.IO;
+using System.Net;
+
+namespace Hyena.Downloader
+{
+ public class HttpDownloader
+ {
+ private object sync_root = new object ();
+ protected object SyncRoot {
+ get { return sync_root; }
+ }
+
+ private HttpWebRequest request;
+ private HttpWebResponse response;
+ private Stream response_stream;
+ private DateTime last_raised_percent_complete;
+ private IAsyncResult async_begin_result;
+
+ public string UserAgent { get; set; }
+ public Uri Uri { get; set; }
+ public TimeSpan ProgressEventRaiseLimit { get; set; }
+ public HttpDownloaderState State { get; private set; }
+
+ private int buffer_size = 8192;
+ public int BufferSize {
+ get { return buffer_size; }
+ set {
+ if (value <= 0) {
+ throw new InvalidOperationException ("Invalid buffer size");
+ }
+ buffer_size = value;
+ }
+ }
+
+ private string name;
+ public string Name {
+ get { return name ?? Path.GetFileName (Uri.UnescapeDataString (Uri.LocalPath)); }
+ set { name = value; }
+ }
+
+ public event Action<HttpDownloader> Started;
+ public event Action<HttpDownloader> Finished;
+ public event Action<HttpDownloader> Progress;
+ public event Action<HttpDownloader> BufferUpdated;
+
+ public HttpDownloader ()
+ {
+ ProgressEventRaiseLimit = TimeSpan.FromSeconds (0.25);
+ }
+
+ public void Start ()
+ {
+ lock (SyncRoot) {
+ if (request != null || async_begin_result != null) {
+ throw new InvalidOperationException ("HttpDownloader is already active");
+ }
+
+ State = new HttpDownloaderState () {
+ Buffer = new Buffer () {
+ Data = new byte[BufferSize]
+ }
+ };
+
+ request = CreateRequest ();
+ async_begin_result = request.BeginGetResponse (OnRequestResponse, this);
+
+ State.StartTime = DateTime.Now;
+ State.Working = true;
+ OnStarted ();
+ }
+ }
+
+ public void Abort ()
+ {
+ lock (SyncRoot) {
+ Close ();
+ OnFinished ();
+ }
+ }
+
+ private void Close ()
+ {
+ lock (SyncRoot) {
+ State.FinishTime = DateTime.Now;
+ State.Working = false;
+
+ if (response_stream != null) {
+ response_stream.Close ();
+ }
+
+ if (response != null) {
+ response.Close ();
+ }
+
+ response_stream = null;
+ response = null;
+ request = null;
+ }
+ }
+
+ protected virtual HttpWebRequest CreateRequest ()
+ {
+ var request = (HttpWebRequest)WebRequest.Create (Uri);
+ request.Method = "GET";
+ request.AllowAutoRedirect = true;
+ request.UserAgent = UserAgent;
+ request.Timeout = 10000;
+ return request;
+ }
+
+ private void OnRequestResponse (IAsyncResult asyncResult)
+ {
+ lock (SyncRoot) {
+ async_begin_result = null;
+
+ if (request == null) {
+ return;
+ }
+
+ var raise = false;
+
+ try {
+ response = (HttpWebResponse)request.EndGetResponse (asyncResult);
+ if (response.StatusCode != HttpStatusCode.OK) {
+ State.Success = false;
+ raise = true;
+ return;
+ }
+
+ State.TotalBytesExpected = response.ContentLength;
+
+ response_stream = response.GetResponseStream ();
+ async_begin_result = response_stream.BeginRead (State.Buffer.Data, 0,
+ State.Buffer.Data.Length, OnResponseRead, this);
+ } catch (Exception e) {
+ State.FailureException = e;
+ State.Success = false;
+ raise = true;
+ } finally {
+ if (raise) {
+ Close ();
+ OnFinished ();
+ }
+ }
+ }
+ }
+
+ private void OnResponseRead (IAsyncResult asyncResult)
+ {
+ lock (SyncRoot) {
+ async_begin_result = null;
+
+ if (request == null || response == null || response_stream == null) {
+ return;
+ }
+
+ try {
+ var now = DateTime.Now;
+
+ State.Buffer.Length = response_stream.EndRead (asyncResult);
+ State.Buffer.TimeStamp = now;
+ State.TotalBytesRead += State.Buffer.Length;
+ State.TransferRate = State.TotalBytesRead / (now - State.StartTime).TotalSeconds;
+ State.PercentComplete = (double)State.TotalBytesRead / (double)State.TotalBytesExpected;
+
+ OnBufferUpdated ();
+
+ if (State.Buffer.Length <= 0) {
+ State.Success = true;
+ Close ();
+ OnFinished ();
+ return;
+ }
+
+ if (State.PercentComplete >= 1 || last_raised_percent_complete == DateTime.MinValue ||
+ (now - last_raised_percent_complete >= ProgressEventRaiseLimit)) {
+ last_raised_percent_complete = now;
+ OnProgress ();
+ }
+
+ async_begin_result = response_stream.BeginRead (State.Buffer.Data, 0,
+ State.Buffer.Data.Length, OnResponseRead, this);
+ } catch (Exception e) {
+ State.FailureException = e;
+ State.Success = false;
+ Close ();
+ OnFinished ();
+ }
+ }
+ }
+
+ protected virtual void OnStarted ()
+ {
+ var handler = Started;
+ if (handler != null) {
+ handler (this);
+ }
+ }
+
+ protected virtual void OnBufferUpdated ()
+ {
+ var handler = BufferUpdated;
+ if (handler != null) {
+ handler (this);
+ }
+ }
+
+ protected virtual void OnProgress ()
+ {
+ var handler = Progress;
+ if (handler != null) {
+ handler (this);
+ }
+ }
+
+ protected virtual void OnFinished ()
+ {
+ var handler = Finished;
+ if (handler != null) {
+ handler (this);
+ }
+ }
+
+ public override string ToString ()
+ {
+ return Name;
+ }
+ }
+}
\ No newline at end of file
diff --git a/Hyena/Hyena.Downloader/HttpDownloaderState.cs b/Hyena/Hyena.Downloader/HttpDownloaderState.cs
new file mode 100644
index 0000000..0cc77b5
--- /dev/null
+++ b/Hyena/Hyena.Downloader/HttpDownloaderState.cs
@@ -0,0 +1,44 @@
+//
+// HttpDownloaderState.cs
+//
+// Author:
+// Aaron Bockover <abockover novell com>
+//
+// Copyright 2010 Novell, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+using System;
+
+namespace Hyena.Downloader
+{
+ public class HttpDownloaderState
+ {
+ public DateTime StartTime { get; internal set; }
+ public DateTime FinishTime { get; internal set; }
+ public double PercentComplete { get; internal set; }
+ public double TransferRate { get; internal set; }
+ public Buffer Buffer { get; internal set; }
+ public long TotalBytesRead { get; internal set; }
+ public long TotalBytesExpected { get; internal set; }
+ public bool Success { get; internal set; }
+ public bool Working { get; internal set; }
+ public Exception FailureException { get; internal set; }
+ }
+}
\ No newline at end of file
diff --git a/Hyena/Hyena.Downloader/HttpFileDownloader.cs b/Hyena/Hyena.Downloader/HttpFileDownloader.cs
new file mode 100644
index 0000000..6c9e4e1
--- /dev/null
+++ b/Hyena/Hyena.Downloader/HttpFileDownloader.cs
@@ -0,0 +1,85 @@
+//
+// HttpFileDownloader.cs
+//
+// Author:
+// Aaron Bockover <abockover novell com>
+//
+// Copyright 2010 Novell, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+using System;
+using System.IO;
+
+namespace Hyena.Downloader
+{
+ public class HttpFileDownloader : HttpDownloader
+ {
+ private FileStream file_stream;
+
+ public string TempPathRoot { get; set; }
+ public string FileExtension { get; set; }
+ public string LocalPath { get; private set; }
+
+ public event Action<HttpFileDownloader> FileFinished;
+
+ public HttpFileDownloader ()
+ {
+ TempPathRoot = System.IO.Path.GetTempPath ();
+ }
+
+ protected override void OnStarted ()
+ {
+ Directory.CreateDirectory (TempPathRoot);
+ LocalPath = Path.Combine (TempPathRoot, CryptoUtil.Md5Encode (Uri.AbsoluteUri));
+ if (!String.IsNullOrEmpty (FileExtension)) {
+ LocalPath += "." + FileExtension;
+ }
+ base.OnStarted ();
+ }
+
+ protected override void OnBufferUpdated ()
+ {
+ if (file_stream == null) {
+ file_stream = new FileStream (LocalPath, FileMode.Create, FileAccess.Write);
+ }
+ file_stream.Write (State.Buffer.Data, 0, (int)State.Buffer.Length);
+ base.OnBufferUpdated ();
+ }
+
+ protected override void OnFinished ()
+ {
+ if (file_stream != null) {
+ file_stream.Close ();
+ file_stream = null;
+ OnFileFinished ();
+ }
+
+ base.OnFinished ();
+ }
+
+ protected virtual void OnFileFinished ()
+ {
+ var handler = FileFinished;
+ if (handler != null) {
+ handler (this);
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/Hyena/Hyena.csproj b/Hyena/Hyena.csproj
index 393f0f2..a73acf4 100644
--- a/Hyena/Hyena.csproj
+++ b/Hyena/Hyena.csproj
@@ -132,6 +132,11 @@
<Compile Include="Hyena\Delegates.cs" />
<Compile Include="Hyena.Collections\LruCache.cs" />
<Compile Include="Hyena.Query\ExactStringQueryValue.cs" />
+ <Compile Include="Hyena.Downloader\Buffer.cs" />
+ <Compile Include="Hyena.Downloader\DownloadManager.cs" />
+ <Compile Include="Hyena.Downloader\HttpDownloader.cs" />
+ <Compile Include="Hyena.Downloader\HttpDownloaderState.cs" />
+ <Compile Include="Hyena.Downloader\HttpFileDownloader.cs" />
<Compile Include="Hyena.Jobs\Job.cs" />
<Compile Include="Hyena.Jobs\JobExtensions.cs" />
<Compile Include="Hyena.Jobs\PriorityHints.cs" />
diff --git a/Hyena/Makefile.am b/Hyena/Makefile.am
index cd40fe6..f76ab62 100644
--- a/Hyena/Makefile.am
+++ b/Hyena/Makefile.am
@@ -37,6 +37,11 @@ SOURCES = \
Hyena.Data/ModelSelection.cs \
Hyena.Data/PropertyStore.cs \
Hyena.Data/SortType.cs \
+ Hyena.Downloader/Buffer.cs \
+ Hyena.Downloader/DownloadManager.cs \
+ Hyena.Downloader/HttpDownloader.cs \
+ Hyena.Downloader/HttpDownloaderState.cs \
+ Hyena.Downloader/HttpFileDownloader.cs \
Hyena.Jobs/Job.cs \
Hyena.Jobs/JobExtensions.cs \
Hyena.Jobs/PriorityHints.cs \
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
