Skip to content

Commit b4c2e21

Browse files
alan-signalalan-signal
committed
Custom streaming video muxer.
1 parent 6080e1f commit b4c2e21

37 files changed

Lines changed: 2017 additions & 166 deletions

‎app/build.gradle‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,8 @@ dependencies {
337337
implementation project(':libsignal-service')
338338
implementation project(':paging')
339339
implementation project(':core-util')
340+
implementation project(':video')
341+
340342
implementation 'org.signal:zkgroup-android:0.7.0'
341343
implementation 'org.whispersystems:signal-client-android:0.1.5'
342344
implementation 'com.google.protobuf:protobuf-javalite:3.10.0'

‎app/src/main/java/org/thoughtcrime/securesms/database/AttachmentDatabase.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -749,7 +749,7 @@ public void updateMessageId(@NonNull Collection<AttachmentId> attachmentIds, lon
749749
}
750750

751751
/**
752-
* @param onlyModifyThisAttachment If false and more than one attachment shares this file, they will all up updated.
752+
* @param onlyModifyThisAttachment If false and more than one attachment shares this file, they will all be updated.
753753
* If true, then guarantees not to affect other attachments.
754754
*/
755755
public void updateAttachmentData(@NonNull DatabaseAttachment databaseAttachment,
@@ -1030,7 +1030,7 @@ public boolean hasStickerAttachments() {
10301030
}
10311031
}
10321032

1033-
private File newFile() throws IOException {
1033+
public File newFile() throws IOException {
10341034
File partsDirectory = context.getDir(DIRECTORY, Context.MODE_PRIVATE);
10351035
return File.createTempFile("part", ".mms", partsDirectory);
10361036
}

‎app/src/main/java/org/thoughtcrime/securesms/jobs/AttachmentCompressionJob.java‎

Lines changed: 72 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,18 @@
55

66
import androidx.annotation.NonNull;
77

8+
import com.google.android.exoplayer2.util.MimeTypes;
9+
810
import org.greenrobot.eventbus.EventBus;
911
import org.signal.core.util.logging.Log;
1012
import org.thoughtcrime.securesms.R;
1113
import org.thoughtcrime.securesms.attachments.Attachment;
1214
import org.thoughtcrime.securesms.attachments.AttachmentId;
1315
import org.thoughtcrime.securesms.attachments.DatabaseAttachment;
16+
import org.thoughtcrime.securesms.crypto.AttachmentSecret;
17+
import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider;
18+
import org.thoughtcrime.securesms.crypto.ModernDecryptingPartInputStream;
19+
import org.thoughtcrime.securesms.crypto.ModernEncryptingPartOutputStream;
1420
import org.thoughtcrime.securesms.database.AttachmentDatabase;
1521
import org.thoughtcrime.securesms.database.DatabaseFactory;
1622
import org.thoughtcrime.securesms.events.PartProgressEvent;
@@ -26,15 +32,22 @@
2632
import org.thoughtcrime.securesms.transport.UndeliverableMessageException;
2733
import org.thoughtcrime.securesms.util.BitmapDecodingException;
2834
import org.thoughtcrime.securesms.util.BitmapUtil;
35+
import org.thoughtcrime.securesms.util.FeatureFlags;
2936
import org.thoughtcrime.securesms.util.MediaUtil;
37+
import org.thoughtcrime.securesms.util.MemoryFileDescriptor;
3038
import org.thoughtcrime.securesms.util.MemoryFileDescriptor.MemoryFileException;
3139
import org.thoughtcrime.securesms.video.InMemoryTranscoder;
32-
import org.thoughtcrime.securesms.video.VideoSizeException;
40+
import org.thoughtcrime.securesms.video.StreamingTranscoder;
41+
import org.thoughtcrime.securesms.video.TranscoderCancelationSignal;
42+
import org.thoughtcrime.securesms.video.TranscoderOptions;
3343
import org.thoughtcrime.securesms.video.VideoSourceException;
3444
import org.thoughtcrime.securesms.video.videoconverter.EncodingException;
3545

3646
import java.io.ByteArrayInputStream;
47+
import java.io.File;
3748
import java.io.IOException;
49+
import java.io.OutputStream;
50+
import java.util.Objects;
3851
import java.util.concurrent.TimeUnit;
3952

4053
public final class AttachmentCompressionJob extends BaseJob {
@@ -177,7 +190,7 @@ private void scaleAndStripExif(@NonNull AttachmentDatabase attachmentDatabase,
177190
@NonNull DatabaseAttachment attachment,
178191
@NonNull MediaConstraints constraints,
179192
@NonNull EventBus eventBus,
180-
@NonNull InMemoryTranscoder.CancelationSignal cancelationSignal)
193+
@NonNull TranscoderCancelationSignal cancelationSignal)
181194
throws UndeliverableMessageException
182195
{
183196
AttachmentDatabase.TransformProperties transformProperties = attachment.getTransformProperties();
@@ -196,34 +209,73 @@ private void scaleAndStripExif(@NonNull AttachmentDatabase attachmentDatabase,
196209
notification.setIndeterminateProgress();
197210

198211
try (MediaDataSource dataSource = attachmentDatabase.mediaDataSourceFor(attachment.getAttachmentId())) {
199-
200212
if (dataSource == null) {
201213
throw new UndeliverableMessageException("Cannot get media data source for attachment.");
202214
}
203215

204216
allowSkipOnFailure = !transformProperties.isVideoEdited();
205-
InMemoryTranscoder.Options options = null;
217+
TranscoderOptions options = null;
206218
if (transformProperties.isVideoTrim()) {
207-
options = new InMemoryTranscoder.Options(transformProperties.getVideoTrimStartTimeUs(), transformProperties.getVideoTrimEndTimeUs());
219+
options = new TranscoderOptions(transformProperties.getVideoTrimStartTimeUs(), transformProperties.getVideoTrimEndTimeUs());
208220
}
209221

210-
try (InMemoryTranscoder transcoder = new InMemoryTranscoder(context, dataSource, options, constraints.getCompressedVideoMaxSize(context))) {
222+
if (FeatureFlags.useStreamingVideoMuxer() || !MemoryFileDescriptor.supported()) {
223+
StreamingTranscoder transcoder = new StreamingTranscoder(dataSource, options, constraints.getCompressedVideoMaxSize(context));
224+
211225
if (transcoder.isTranscodeRequired()) {
212-
MediaStream mediaStream = transcoder.transcode(percent -> {
213-
notification.setProgress(100, percent);
214-
eventBus.postSticky(new PartProgressEvent(attachment,
215-
PartProgressEvent.Type.COMPRESSION,
216-
100,
217-
percent));
218-
}, cancelationSignal);
219-
220-
attachmentDatabase.updateAttachmentData(attachment, mediaStream, transformProperties.isVideoEdited());
226+
Log.i(TAG, "Compressing with streaming muxer");
227+
AttachmentSecret attachmentSecret = AttachmentSecretProvider.getInstance(context).getOrCreateAttachmentSecret();
228+
229+
File file = DatabaseFactory.getAttachmentDatabase(context)
230+
.newFile();
231+
file.deleteOnExit();
232+
233+
try {
234+
try (OutputStream outputStream = ModernEncryptingPartOutputStream.createFor(attachmentSecret, file, true).second) {
235+
transcoder.transcode(percent -> {
236+
notification.setProgress(100, percent);
237+
eventBus.postSticky(new PartProgressEvent(attachment,
238+
PartProgressEvent.Type.COMPRESSION,
239+
100,
240+
percent));
241+
}, outputStream, cancelationSignal);
242+
}
243+
244+
MediaStream mediaStream = new MediaStream(ModernDecryptingPartInputStream.createFor(attachmentSecret, file, 0), MimeTypes.VIDEO_MP4, 0, 0);
245+
attachmentDatabase.updateAttachmentData(attachment, mediaStream, transformProperties.isVideoEdited());
246+
} finally {
247+
if (!file.delete()) {
248+
Log.w(TAG, "Failed to delete temp file");
249+
}
250+
}
251+
221252
attachmentDatabase.markAttachmentAsTransformed(attachment.getAttachmentId());
222-
DatabaseAttachment updatedAttachment = attachmentDatabase.getAttachment(attachment.getAttachmentId());
223-
if (updatedAttachment == null) {
224-
throw new AssertionError();
253+
254+
return Objects.requireNonNull(attachmentDatabase.getAttachment(attachment.getAttachmentId()));
255+
} else {
256+
Log.i(TAG, "Transcode was not required");
257+
}
258+
} else {
259+
try (InMemoryTranscoder transcoder = new InMemoryTranscoder(context, dataSource, options, constraints.getCompressedVideoMaxSize(context))) {
260+
if (transcoder.isTranscodeRequired()) {
261+
Log.i(TAG, "Compressing with android in-memory muxer");
262+
263+
MediaStream mediaStream = transcoder.transcode(percent -> {
264+
notification.setProgress(100, percent);
265+
eventBus.postSticky(new PartProgressEvent(attachment,
266+
PartProgressEvent.Type.COMPRESSION,
267+
100,
268+
percent));
269+
}, cancelationSignal);
270+
271+
attachmentDatabase.updateAttachmentData(attachment, mediaStream, transformProperties.isVideoEdited());
272+
273+
attachmentDatabase.markAttachmentAsTransformed(attachment.getAttachmentId());
274+
275+
return Objects.requireNonNull(attachmentDatabase.getAttachment(attachment.getAttachmentId()));
276+
} else {
277+
Log.i(TAG, "Transcode was not required (in-memory transcoder)");
225278
}
226-
return updatedAttachment;
227279
}
228280
}
229281
}
@@ -237,7 +289,7 @@ private void scaleAndStripExif(@NonNull AttachmentDatabase attachmentDatabase,
237289
throw new UndeliverableMessageException("Failed to transcode and cannot skip due to editing", e);
238290
}
239291
}
240-
} catch (IOException | MmsException | VideoSizeException e) {
292+
} catch (IOException | MmsException e) {
241293
throw new UndeliverableMessageException("Failed to transcode", e);
242294
}
243295
return attachment;

‎app/src/main/java/org/thoughtcrime/securesms/mms/MediaConstraints.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.thoughtcrime.securesms.attachments.Attachment;
1212
import org.thoughtcrime.securesms.util.BitmapDecodingException;
1313
import org.thoughtcrime.securesms.util.BitmapUtil;
14+
import org.thoughtcrime.securesms.util.FeatureFlags;
1415
import org.thoughtcrime.securesms.util.MediaUtil;
1516
import org.thoughtcrime.securesms.util.MemoryFileDescriptor;
1617

@@ -76,6 +77,6 @@ public boolean canResize(@NonNull Attachment attachment) {
7677
}
7778

7879
public static boolean isVideoTranscodeAvailable() {
79-
return Build.VERSION.SDK_INT >= 26 && MemoryFileDescriptor.supported();
80+
return Build.VERSION.SDK_INT >= 26 && (FeatureFlags.useStreamingVideoMuxer() || MemoryFileDescriptor.supported());
8081
}
8182
}

‎app/src/main/java/org/thoughtcrime/securesms/util/FeatureFlags.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ public final class FeatureFlags {
6565
private static final String GV1_FORCED_MIGRATE = "android.groupsV1Migration.forced";
6666
private static final String GV1_MIGRATION_JOB = "android.groupsV1Migration.job";
6767
private static final String SEND_VIEWED_RECEIPTS = "android.sendViewedReceipts";
68+
private static final String DISABLE_CUSTOM_VIDEO_MUXER = "android.disableCustomVideoMuxer";
6869

6970
/**
7071
* We will only store remote values for flags in this set. If you want a flag to be controllable
@@ -108,7 +109,8 @@ public final class FeatureFlags {
108109
VERIFY_V2,
109110
CLIENT_EXPIRATION,
110111
GROUP_CALLING,
111-
GV1_MIGRATION_JOB
112+
GV1_MIGRATION_JOB,
113+
DISABLE_CUSTOM_VIDEO_MUXER
112114
);
113115

114116
/**
@@ -253,6 +255,11 @@ public static boolean sendViewedReceipts() {
253255
return getBoolean(SEND_VIEWED_RECEIPTS, false);
254256
}
255257

258+
/** Whether to use the custom streaming muxer or built in android muxer. */
259+
public static boolean useStreamingVideoMuxer() {
260+
return !getBoolean(DISABLE_CUSTOM_VIDEO_MUXER, false);
261+
}
262+
256263
/** Only for rendering debug info. */
257264
public static synchronized @NonNull Map<String, Object> getMemoryValues() {
258265
return new TreeMap<>(REMOTE_VALUES);

‎app/src/main/java/org/thoughtcrime/securesms/video/InMemoryTranscoder.java‎

Lines changed: 3 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,14 @@ public final class InMemoryTranscoder implements Closeable {
3939
private final long memoryFileEstimate;
4040
private final boolean transcodeRequired;
4141
private final long fileSizeEstimate;
42-
private final @Nullable Options options;
42+
private final @Nullable TranscoderOptions options;
4343

4444
private @Nullable MemoryFileDescriptor memoryFile;
4545

4646
/**
4747
* @param upperSizeLimit A upper size to transcode to. The actual output size can be up to 10% smaller.
4848
*/
49-
public InMemoryTranscoder(@NonNull Context context, @NonNull MediaDataSource dataSource, @Nullable Options options, long upperSizeLimit) throws IOException, VideoSourceException {
49+
public InMemoryTranscoder(@NonNull Context context, @NonNull MediaDataSource dataSource, @Nullable TranscoderOptions options, long upperSizeLimit) throws IOException, VideoSourceException {
5050
this.context = context;
5151
this.dataSource = dataSource;
5252
this.options = options;
@@ -75,7 +75,7 @@ public InMemoryTranscoder(@NonNull Context context, @NonNull MediaDataSource dat
7575
}
7676

7777
public @NonNull MediaStream transcode(@NonNull Progress progress,
78-
@Nullable CancelationSignal cancelationSignal)
78+
@Nullable TranscoderCancelationSignal cancelationSignal)
7979
throws IOException, EncodingException, VideoSizeException
8080
{
8181
if (memoryFile != null) throw new AssertionError("Not expecting to reuse transcoder");
@@ -202,18 +202,4 @@ private static boolean containsLocation(MediaMetadataRetriever mediaMetadataRetr
202202
public interface Progress {
203203
void onProgress(int percent);
204204
}
205-
206-
public interface CancelationSignal {
207-
boolean isCanceled();
208-
}
209-
210-
public final static class Options {
211-
final long startTimeUs;
212-
final long endTimeUs;
213-
214-
public Options(long startTimeUs, long endTimeUs) {
215-
this.startTimeUs = startTimeUs;
216-
this.endTimeUs = endTimeUs;
217-
}
218-
}
219205
}

0 commit comments

Comments
 (0)