[fractal] session: Send media info with attachments
- From: Marge Bot <marge-bot src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [fractal] session: Send media info with attachments
- Date: Mon, 10 Oct 2022 09:15:00 +0000 (UTC)
commit dd1f5b8246f066b538cd5823d89490bcb3cd88f4
Author: Kévin Commaille <zecakeh tedomum fr>
Date: Mon Oct 3 11:57:37 2022 +0200
session: Send media info with attachments
Generate thumbnails for images.
Part-of: <https://gitlab.gnome.org/GNOME/fractal/-/merge_requests/1173>
Cargo.lock | 128 ++++++++++++++++++++++++++++++--
Cargo.toml | 10 ++-
src/session/content/room_history/mod.rs | 63 ++++++++++++----
src/session/room/mod.rs | 37 ++++++++-
src/utils/media.rs | 89 ++++++++++++++++++++++
5 files changed, 301 insertions(+), 26 deletions(-)
---
diff --git a/Cargo.lock b/Cargo.lock
index 417789b6f..d3acaa4d2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1005,6 +1005,16 @@ dependencies = [
"rustc_version",
]
+[[package]]
+name = "flate2"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide 0.5.4",
+]
+
[[package]]
name = "fnv"
version = "1.0.7"
@@ -1047,12 +1057,13 @@ dependencies = [
"gst-plugin-gtk4",
"gstreamer",
"gstreamer-base",
+ "gstreamer-pbutils",
"gstreamer-player",
"gstreamer-video",
"gtk-macros",
"gtk4",
"html2pango",
- "image",
+ "image 0.23.14",
"indexmap",
"libadwaita",
"libsecret",
@@ -1653,6 +1664,20 @@ dependencies = [
"thiserror",
]
+[[package]]
+name = "gstreamer-audio-sys"
+version = "0.18.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "a34258fb53c558c0f41dad194037cbeaabf49d347570df11b8bd1c4897cf7d7c"
+dependencies = [
+ "glib-sys",
+ "gobject-sys",
+ "gstreamer-base-sys",
+ "gstreamer-sys",
+ "libc",
+ "system-deps",
+]
+
[[package]]
name = "gstreamer-base"
version = "0.18.0"
@@ -1680,6 +1705,35 @@ dependencies = [
"system-deps",
]
+[[package]]
+name = "gstreamer-pbutils"
+version = "0.18.7"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "330684c49f79775d7acce8bef5a7a7475f02374c9c6cead39ced3ad423fc8ea9"
+dependencies = [
+ "bitflags",
+ "glib",
+ "gstreamer",
+ "gstreamer-pbutils-sys",
+ "libc",
+ "thiserror",
+]
+
+[[package]]
+name = "gstreamer-pbutils-sys"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "36f79839066fbcc6d1a8690b2f85d5cc5cdc0984f36d4054f5cc67a7ad3ab72d"
+dependencies = [
+ "glib-sys",
+ "gobject-sys",
+ "gstreamer-audio-sys",
+ "gstreamer-sys",
+ "gstreamer-video-sys",
+ "libc",
+ "system-deps",
+]
+
[[package]]
name = "gstreamer-player"
version = "0.18.0"
@@ -2077,13 +2131,31 @@ dependencies = [
"byteorder",
"color_quant",
"gif",
- "jpeg-decoder",
+ "jpeg-decoder 0.1.22",
"num-iter",
"num-rational 0.3.2",
"num-traits",
- "png",
+ "png 0.16.8",
"scoped_threadpool",
- "tiff",
+ "tiff 0.6.1",
+]
+
+[[package]]
+name = "image"
+version = "0.24.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "bd8e4fb07cf672b1642304e731ef8a6a4c7891d67bb4fd4f5ce58cd6ed86803c"
+dependencies = [
+ "bytemuck",
+ "byteorder",
+ "color_quant",
+ "gif",
+ "jpeg-decoder 0.2.6",
+ "num-rational 0.4.1",
+ "num-traits",
+ "png 0.17.6",
+ "scoped_threadpool",
+ "tiff 0.7.3",
]
[[package]]
@@ -2163,6 +2235,15 @@ dependencies = [
"rayon",
]
+[[package]]
+name = "jpeg-decoder"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "9478aa10f73e7528198d75109c8be5cd7d15fb530238040148d5f9a22d4c5b3b"
+dependencies = [
+ "rayon",
+]
+
[[package]]
name = "js-sys"
version = "0.3.60"
@@ -2478,6 +2559,7 @@ dependencies = [
"futures-signals",
"futures-util",
"http",
+ "image 0.24.4",
"matrix-sdk-base",
"matrix-sdk-common",
"matrix-sdk-indexeddb",
@@ -2713,6 +2795,15 @@ dependencies = [
"autocfg",
]
+[[package]]
+name = "miniz_oxide"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"
+dependencies = [
+ "adler",
+]
+
[[package]]
name = "mio"
version = "0.8.4"
@@ -3290,6 +3381,18 @@ dependencies = [
"miniz_oxide 0.3.7",
]
+[[package]]
+name = "png"
+version = "0.17.6"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "8f0e7f4c94ec26ff209cee506314212639d6c91b80afb82984819fafce9df01c"
+dependencies = [
+ "bitflags",
+ "crc32fast",
+ "flate2",
+ "miniz_oxide 0.5.4",
+]
+
[[package]]
name = "polling"
version = "2.3.0"
@@ -3434,7 +3537,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "16d2f1455f3630c6e5107b4f2b94e74d76dea80736de0981fd27644216cff57f"
dependencies = [
"checked_int_cast",
- "image",
+ "image 0.23.14",
]
[[package]]
@@ -3660,7 +3763,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "6fa79947f53b20adb909a323d828d0fd744fa9d854792df07913b083bcd4d63b"
dependencies = [
"g2p",
- "image",
+ "image 0.23.14",
"lru 0.6.6",
]
@@ -4295,11 +4398,22 @@ version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "9a53f4706d65497df0c4349241deddf35f84cee19c87ed86ea8ca590f4464437"
dependencies = [
- "jpeg-decoder",
+ "jpeg-decoder 0.1.22",
"miniz_oxide 0.4.4",
"weezl",
]
+[[package]]
+name = "tiff"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "7259662e32d1e219321eb309d5f9d898b779769d81b76e762c07c8e5d38fcb65"
+dependencies = [
+ "flate2",
+ "jpeg-decoder 0.2.6",
+ "weezl",
+]
+
[[package]]
name = "tinyvec"
version = "1.6.0"
diff --git a/Cargo.toml b/Cargo.toml
index 8a255d9ac..07680938c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -44,6 +44,7 @@ gst_base = { version = "0.18", package = "gstreamer-base" }
gst_video = { version = "0.18", package = "gstreamer-video" }
gst_player = { version = "0.18", package = "gstreamer-player" }
gst_gtk = { version = "0.1.0", package = "gst-plugin-gtk4" }
+gst_pbutils = { version = "0.18", package = "gstreamer-pbutils" }
image = { version = "0.23", default-features = false, features = ["png"] }
regex = "1.5.4"
mime_guess = "2.0.3"
@@ -74,7 +75,14 @@ version = "0.1.1"
[dependencies.matrix-sdk]
version = "0.6.0"
-features = ["socks", "sso-login", "markdown", "qrcode", "experimental-timeline"]
+features = [
+ "socks",
+ "sso-login",
+ "markdown",
+ "qrcode",
+ "experimental-timeline",
+ "image-rayon",
+]
[dependencies.ruma]
version = "0.7.4"
diff --git a/src/session/content/room_history/mod.rs b/src/session/content/room_history/mod.rs
index b47944565..665861bed 100644
--- a/src/session/content/room_history/mod.rs
+++ b/src/session/content/room_history/mod.rs
@@ -23,14 +23,17 @@ use gtk::{
CompositeTemplate,
};
use log::{error, warn};
-use matrix_sdk::ruma::{
- events::{
- room::message::{
- EmoteMessageEventContent, FormattedBody, MessageType, TextMessageEventContent,
+use matrix_sdk::{
+ attachment::{AttachmentInfo, BaseFileInfo, BaseImageInfo},
+ ruma::{
+ events::{
+ room::message::{
+ EmoteMessageEventContent, FormattedBody, MessageType, TextMessageEventContent,
+ },
+ AnySyncMessageLikeEvent, AnySyncTimelineEvent, SyncMessageLikeEvent,
},
- AnySyncMessageLikeEvent, AnySyncTimelineEvent, SyncMessageLikeEvent,
+ EventId,
},
- EventId,
};
use ruma::events::{
room::message::{ForwardThread, LocationMessageEventContent, RoomMessageEventContent},
@@ -52,7 +55,10 @@ use crate::{
user::UserExt,
},
spawn, spawn_tokio, toast,
- utils::{media::filename_for_mime, template_callbacks::TemplateCallbacks},
+ utils::{
+ media::{filename_for_mime, get_audio_info, get_image_info, get_video_info},
+ template_callbacks::TemplateCallbacks,
+ },
};
#[derive(Debug, Hash, Eq, PartialEq, Clone, Copy, glib::Enum)]
@@ -969,11 +975,15 @@ impl RoomHistory {
}
if let Some(room) = self.room() {
- room.send_attachment(
- image.save_to_png_bytes().to_vec(),
- mime::IMAGE_PNG,
- &filename,
- );
+ let bytes = image.save_to_png_bytes();
+ let info = AttachmentInfo::Image(BaseImageInfo {
+ width: Some((image.width() as u32).into()),
+ height: Some((image.height() as u32).into()),
+ size: Some((bytes.len() as u32).into()),
+ blurhash: None,
+ });
+
+ room.send_attachment(bytes.to_vec(), mime::IMAGE_PNG, &filename, info);
}
}
@@ -1008,6 +1018,7 @@ impl RoomHistory {
let attributes: &[&str] = &[
*gio::FILE_ATTRIBUTE_STANDARD_CONTENT_TYPE,
*gio::FILE_ATTRIBUTE_STANDARD_DISPLAY_NAME,
+ *gio::FILE_ATTRIBUTE_STANDARD_SIZE,
];
// Read mime type.
@@ -1025,10 +1036,15 @@ impl RoomHistory {
.and_then(|info| info.content_type())
.and_then(|content_type| mime::Mime::from_str(&content_type).ok())
.unwrap_or(mime::APPLICATION_OCTET_STREAM);
- let filename = info.map(|info| info.display_name()).map_or_else(
+ let filename = info.as_ref().map(|info| info.display_name()).map_or_else(
|| filename_for_mime(Some(mime.as_ref()), None),
|name| name.to_string(),
);
+ let size = info
+ .as_ref()
+ .map(|info| info.size())
+ .filter(|size| *size > 0)
+ .map(|size| (size as u32).into());
match file.load_contents_future().await {
Ok((bytes, _tag)) => {
@@ -1040,7 +1056,26 @@ impl RoomHistory {
}
if let Some(room) = self.room() {
- room.send_attachment(bytes.clone(), mime.clone(), &filename);
+ let info = match mime.type_() {
+ mime::IMAGE => {
+ let mut info = get_image_info(&file).await;
+ info.size = size;
+ AttachmentInfo::Image(info)
+ }
+ mime::VIDEO => {
+ let mut info = get_video_info(&file).await;
+ info.size = size;
+ AttachmentInfo::Video(info)
+ }
+ mime::AUDIO => {
+ let mut info = get_audio_info(&file).await;
+ info.size = size;
+ AttachmentInfo::Audio(info)
+ }
+ _ => AttachmentInfo::File(BaseFileInfo { size }),
+ };
+
+ room.send_attachment(bytes.clone(), mime.clone(), &filename, info);
}
}
Err(err) => {
diff --git a/src/session/room/mod.rs b/src/session/room/mod.rs
index efbd5e09f..4d171abf5 100644
--- a/src/session/room/mod.rs
+++ b/src/session/room/mod.rs
@@ -10,13 +10,13 @@ mod reaction_list;
mod room_type;
mod timeline;
-use std::{cell::RefCell, path::PathBuf};
+use std::{cell::RefCell, io::Cursor, path::PathBuf};
use gettextrs::{gettext, ngettext};
use gtk::{glib, glib::clone, prelude::*, subclass::prelude::*};
use log::{debug, error, info, warn};
use matrix_sdk::{
- attachment::AttachmentConfig,
+ attachment::{generate_image_thumbnail, AttachmentConfig, AttachmentInfo, Thumbnail},
deserialized_responses::{JoinedRoom, LeftRoom, SyncTimelineEvent},
room::Room as MatrixRoom,
ruma::{
@@ -1569,13 +1569,42 @@ impl Room {
Some(())
}
- pub fn send_attachment(&self, bytes: Vec<u8>, mime: mime::Mime, body: &str) {
+ pub fn send_attachment(
+ &self,
+ bytes: Vec<u8>,
+ mime: mime::Mime,
+ body: &str,
+ info: AttachmentInfo,
+ ) {
let matrix_room = self.matrix_room();
if let MatrixRoom::Joined(matrix_room) = matrix_room {
let body = body.to_string();
spawn_tokio!(async move {
- let config = AttachmentConfig::default();
+ // Needed to hold the thumbnail data until it is sent.
+ let data_slot;
+
+ // The method will filter compatible mime types so we don't need to
+ // since we ignore errors.
+ let thumbnail = match generate_image_thumbnail(&mime, Cursor::new(&bytes), None) {
+ Ok((data, info)) => {
+ data_slot = data;
+ Some(Thumbnail {
+ data: &data_slot,
+ content_type: &mime::IMAGE_JPEG,
+ info: Some(info),
+ })
+ }
+ _ => None,
+ };
+
+ let config = if let Some(thumbnail) = thumbnail {
+ AttachmentConfig::with_thumbnail(thumbnail)
+ } else {
+ AttachmentConfig::new()
+ }
+ .info(info);
+
matrix_room
// TODO This should be added to pending messages instead of
// sending it directly.
diff --git a/src/utils/media.rs b/src/utils/media.rs
index ba8be9a99..dfc08ca20 100644
--- a/src/utils/media.rs
+++ b/src/utils/media.rs
@@ -1,6 +1,10 @@
//! Collection of methods for media files.
+use std::{cell::Cell, sync::Mutex};
+
use gettextrs::gettext;
+use gtk::{gdk_pixbuf, gio, prelude::*};
+use matrix_sdk::attachment::{BaseAudioInfo, BaseImageInfo, BaseVideoInfo};
use ruma::events::room::MediaSource;
/// Get the unique id of the given `MediaSource`.
@@ -56,3 +60,88 @@ pub fn filename_for_mime(mime_type: Option<&str>, fallback: Option<mime::Name>)
.map(|extension| format!("{}.{}", name, extension))
.unwrap_or(name)
}
+
+pub async fn get_image_info(file: &gio::File) -> BaseImageInfo {
+ let mut info = BaseImageInfo {
+ width: None,
+ height: None,
+ size: None,
+ blurhash: None,
+ };
+
+ let path = match file.path() {
+ Some(path) => path,
+ None => return info,
+ };
+
+ if let Ok(Some((_format, w, h))) = gdk_pixbuf::Pixbuf::file_info_future(path).await {
+ info.width = Some((w as u32).into());
+ info.height = Some((h as u32).into());
+ }
+
+ info
+}
+
+async fn get_gstreamer_media_info(file: &gio::File) -> Option<gst_pbutils::DiscovererInfo> {
+ let timeout = gst::ClockTime::from_seconds(15);
+ let discoverer = gst_pbutils::Discoverer::new(timeout).ok()?;
+
+ let (sender, receiver) = futures::channel::oneshot::channel();
+ let sender = Mutex::new(Cell::new(Some(sender)));
+ discoverer.connect_discovered(move |_, info, _| {
+ if let Some(sender) = sender.lock().unwrap().take() {
+ sender.send(info.clone()).unwrap();
+ }
+ });
+
+ discoverer.start();
+ discoverer.discover_uri_async(&file.uri()).ok()?;
+
+ let media_info = receiver.await.unwrap();
+ discoverer.stop();
+
+ Some(media_info)
+}
+
+pub async fn get_video_info(file: &gio::File) -> BaseVideoInfo {
+ let mut info = BaseVideoInfo {
+ duration: None,
+ width: None,
+ height: None,
+ size: None,
+ blurhash: None,
+ };
+
+ let media_info = match get_gstreamer_media_info(file).await {
+ Some(media_info) => media_info,
+ None => return info,
+ };
+
+ info.duration = media_info.duration().map(Into::into);
+
+ if let Some(stream_info) = media_info
+ .video_streams()
+ .get(0)
+ .and_then(|s| s.downcast_ref::<gst_pbutils::DiscovererVideoInfo>())
+ {
+ info.width = Some(stream_info.width().into());
+ info.height = Some(stream_info.height().into());
+ }
+
+ info
+}
+
+pub async fn get_audio_info(file: &gio::File) -> BaseAudioInfo {
+ let mut info = BaseAudioInfo {
+ duration: None,
+ size: None,
+ };
+
+ let media_info = match get_gstreamer_media_info(file).await {
+ Some(media_info) => media_info,
+ None => return info,
+ };
+
+ info.duration = media_info.duration().map(Into::into);
+ info
+}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
