Added #1893 Experimental playback of DefineVideoStream tags with VLC player

This commit is contained in:
Jindra Petřík
2022-12-04 21:56:16 +01:00
parent efded83357
commit 85668bbb07
23 changed files with 674 additions and 35 deletions

View File

@@ -2187,7 +2187,7 @@ public final class SWF implements SWFContainerItem, Timelined, Openable {
}
}
public static void populateVideoFrames(int streamId, Iterable<Tag> tags, HashMap<Integer, VideoFrameTag> output) {
public static void populateVideoFrames(int streamId, Iterable<Tag> tags, Map<Integer, VideoFrameTag> output) {
for (Tag t : tags) {
if (t instanceof VideoFrameTag) {
VideoFrameTag videoFrameTag = (VideoFrameTag) t;

View File

@@ -818,6 +818,11 @@ public final class Configuration {
@ConfigurationName("gui.scale")
public static ConfigurationItem<Double> uiScale = null;
@ConfigurationDefaultString("")
@ConfigurationCategory("paths")
@ConfigurationDirectory
public static ConfigurationItem<String> vlcPlayerLocation = null;
private enum OSId {
WINDOWS, OSX, UNIX
}

View File

@@ -43,6 +43,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
*
@@ -102,8 +103,12 @@ public class MovieExporter {
}
public byte[] exportMovie(DefineVideoStreamTag videoStream, MovieExportMode mode) throws IOException {
return exportMovie(videoStream, mode, false);
}
public byte[] exportMovie(DefineVideoStreamTag videoStream, MovieExportMode mode, boolean ffdecInternal) throws IOException {
SWF swf = videoStream.getSwf();
HashMap<Integer, VideoFrameTag> frames = new HashMap<>();
Map<Integer, VideoFrameTag> frames = new HashMap<>();
SWF.populateVideoFrames(videoStream.characterID, swf.getTags(), frames);
if (frames.isEmpty()) {
return SWFInputStream.BYTE_ARRAY_EMPTY;
@@ -120,10 +125,14 @@ public class MovieExporter {
int verticalAdjustment = 0;
int[] frameNumArray = Helper.toIntArray(frames.keySet());
Arrays.sort(frameNumArray);
FLVTAG lastTag = null;
int frameNum = 0;
for (int i = 0; i < frameNumArray.length; i++) {
VideoFrameTag tag = frames.get(frameNumArray[i]);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
frameNum = frameNumArray[i];
int frameType = 1;
if ((videoStream.codecID == DefineVideoStreamTag.CODEC_VP6)
@@ -190,7 +199,11 @@ public class MovieExporter {
}
baos.write(tag.videoData.getRangeData());
flv.writeTag(new FLVTAG((int) Math.floor(i * 1000.0 / swf.frameRate), new VIDEODATA(frameType, videoStream.codecID, baos.toByteArray())));
flv.writeTag(lastTag = new FLVTAG((long)Math.floor(ffdecInternal ? frameNum * 5000.0 : (frameNum * 1000.0 / swf.frameRate)), new VIDEODATA(frameType, videoStream.codecID, baos.toByteArray())));
}
if (ffdecInternal && lastTag != null) {
lastTag.timeStamp = frameNum * 5000 + 5000;
flv.writeTag(lastTag);
}
return fos.toByteArray();
}

View File

@@ -16,26 +16,63 @@
*/
package com.jpexs.decompiler.flash.tags;
import com.jpexs.decompiler.flash.ReadOnlyTagList;
import com.jpexs.decompiler.flash.SWF;
import com.jpexs.decompiler.flash.SWFInputStream;
import com.jpexs.decompiler.flash.SWFOutputStream;
import com.jpexs.decompiler.flash.configuration.Configuration;
import com.jpexs.decompiler.flash.exporters.MovieExporter;
import com.jpexs.decompiler.flash.exporters.commonshape.ExportRectangle;
import com.jpexs.decompiler.flash.exporters.commonshape.Matrix;
import com.jpexs.decompiler.flash.exporters.commonshape.SVGExporter;
import com.jpexs.decompiler.flash.exporters.modes.MovieExportMode;
import com.jpexs.decompiler.flash.tags.base.BoundedTag;
import com.jpexs.decompiler.flash.tags.base.CharacterTag;
import com.jpexs.decompiler.flash.tags.base.ButtonTag;
import com.jpexs.decompiler.flash.tags.base.DrawableTag;
import com.jpexs.decompiler.flash.tags.base.RenderContext;
import com.jpexs.decompiler.flash.timeline.Timeline;
import com.jpexs.decompiler.flash.timeline.Timelined;
import com.jpexs.decompiler.flash.types.BasicType;
import com.jpexs.decompiler.flash.types.ColorTransform;
import com.jpexs.decompiler.flash.types.MATRIX;
import com.jpexs.decompiler.flash.types.RECT;
import com.jpexs.decompiler.flash.types.SOUNDINFO;
import com.jpexs.decompiler.flash.types.annotations.Internal;
import com.jpexs.decompiler.flash.types.annotations.Reserved;
import com.jpexs.decompiler.flash.types.annotations.SWFType;
import com.jpexs.decompiler.flash.types.annotations.SWFVersion;
import com.jpexs.decompiler.flash.types.filters.BlendComposite;
import com.jpexs.helpers.ByteArrayRange;
import com.jpexs.helpers.Helper;
import com.jpexs.helpers.SerializableImage;
import com.jpexs.video.FrameListener;
import com.jpexs.video.SimpleMediaPlayer;
import java.awt.AlphaComposite;
import java.awt.BasicStroke;
import java.awt.Color;
import java.awt.Graphics2D;
import java.awt.Rectangle;
import java.awt.Shape;
import java.awt.geom.AffineTransform;
import java.awt.geom.Rectangle2D;
import java.awt.image.BufferedImage;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
*
* @author JPEXS
*/
@SWFVersion(from = 6)
public class DefineVideoStreamTag extends CharacterTag implements BoundedTag {
public class DefineVideoStreamTag extends DrawableTag implements BoundedTag, Timelined {
public static final int ID = 60;
@@ -65,6 +102,22 @@ public class DefineVideoStreamTag extends CharacterTag implements BoundedTag {
@SWFType(BasicType.UI8)
public int codecID;
@Internal
private SimpleMediaPlayer mediaPlayer;
@Internal
private final Object getFrameLock = new Object();
@Internal
private BufferedImage activeFrame = null;
@Internal
private ReadOnlyTagList tags;
@Internal
private int lastRatio = -1;
private Timeline timeline;
public static final int CODEC_SORENSON_H263 = 2;
public static final int CODEC_SCREEN_VIDEO = 3;
@@ -149,4 +202,219 @@ public class DefineVideoStreamTag extends CharacterTag implements BoundedTag {
public RECT getRectWithStrokes() {
return getRect();
}
private void initPlayer() {
if (mediaPlayer != null) { // && !mediaPlayer.isFinished()) {
return;
}
mediaPlayer = new SimpleMediaPlayer();
mediaPlayer.addFrameListener(new FrameListener() {
@Override
public void newFrameRecieved(BufferedImage image) {
synchronized (getFrameLock) {
activeFrame = image;
//System.out.println("received frame");
getFrameLock.notifyAll();
/*if (mediaPlayer.isFinished()) {
mediaPlayer.rewind();
}*/
}
}
});
MovieExporter exp = new MovieExporter();
try {
byte[] data = exp.exportMovie(this, MovieExportMode.FLV, true);
File tempFile = File.createTempFile("ffdec_video", ".flv");
tempFile.deleteOnExit();
Helper.writeFile(tempFile.getAbsolutePath(), data);
mediaPlayer.play(tempFile.getAbsolutePath());
//mediaPlayer.pause();
} catch (IOException ex) {
Logger.getLogger(DefineVideoStreamTag.class.getName()).log(Level.SEVERE, null, ex);
}
}
@Override
public int getUsedParameters() {
return PARAMETER_RATIO;
}
@Override
public Shape getOutline(int frame, int time, int ratio, RenderContext renderContext, Matrix transformation, boolean stroked) {
return transformation.toTransform().createTransformedShape(new Rectangle2D.Double(0, 0, width * SWF.unitDivisor, height * SWF.unitDivisor));
}
@Override
public synchronized void toImage(int frame, int time, int ratio, RenderContext renderContext, SerializableImage image, SerializableImage fullImage, boolean isClip, Matrix transformation, Matrix prevTransformation, Matrix absoluteTransformation, Matrix fullTransformation, ColorTransform colorTransform, double unzoom, boolean sameImage, ExportRectangle viewRect, boolean scaleStrokes, int drawMode, int blendMode) {
if (!Configuration.vlcPlayerLocation.hasValue() || !new File(Configuration.vlcPlayerLocation.get()).exists()) {
Graphics2D g = (Graphics2D) image.getBufferedImage().getGraphics();
Matrix mat = transformation;
AffineTransform trans = mat.preConcatenate(Matrix.getScaleInstance(1 / SWF.unitDivisor)).toTransform();
g.setTransform(trans);
BoundedTag b = (BoundedTag) this;
g.setPaint(new Color(255, 255, 255, 128));
g.setComposite(BlendComposite.Invert);
g.setStroke(new BasicStroke((int) SWF.unitDivisor));
RECT r = b.getRect();
g.setFont(g.getFont().deriveFont((float) (12 * SWF.unitDivisor)));
g.drawString(toString(), r.Xmin + (int) (3 * SWF.unitDivisor), r.Ymin + (int) (15 * SWF.unitDivisor));
g.draw(new Rectangle(r.Xmin, r.Ymin, r.getWidth(), r.getHeight()));
g.drawLine(r.Xmin, r.Ymin, r.Xmax, r.Ymax);
g.drawLine(r.Xmax, r.Ymin, r.Xmin, r.Ymax);
g.setComposite(AlphaComposite.Dst);
return;
}
synchronized (DefineVideoStreamTag.class) {
if (!(activeFrame != null && lastRatio == ratio)) {
synchronized (getFrameLock) {
activeFrame = null;
getFrameLock.notifyAll();
}
initPlayer();
if (ratio == -1) {
ratio = 0;
}
//float oneFr = 1f / getNumFrames();
ratio--;
if (mediaPlayer.isFinished()) {
return;
}
mediaPlayer.setPosition((float) ratio / getNumFrames());
try {
synchronized (getFrameLock) {
if (activeFrame == null) {
//System.out.println("waiting...");
getFrameLock.wait();
//System.out.println("awakened");
}
}
} catch (InterruptedException ex) {
//Logger.getLogger(DefineVideoStreamTag.class.getName()).log(Level.SEVERE, null, ex);
}
}
synchronized (getFrameLock) {
if (activeFrame != null) {
//System.out.println("drawed");
Graphics2D graphics = (Graphics2D) image.getBufferedImage().getGraphics();
AffineTransform at = transformation.toTransform();
at.preConcatenate(AffineTransform.getScaleInstance(1 / SWF.unitDivisor, 1 / SWF.unitDivisor));
graphics.setTransform(at);
//Point p = transformation.inverse().transform(0, 0);
graphics.drawImage(activeFrame, 0, 0,
(int) Math.round(width * SWF.unitDivisor),
(int) Math.round(height * SWF.unitDivisor),
0, 0, width, height,
null);
}
}
}
//System.out.println("toImage return");
}
@Override
public void toSVG(SVGExporter exporter, int ratio, ColorTransform colorTransform, int level) throws IOException {
}
@Override
public void toHtmlCanvas(StringBuilder result, double unitDivisor) {
}
@Override
public int getNumFrames() {
return numFrames;
}
@Override
public boolean isSingleFrame() {
return getNumFrames() == 1;
}
@Override
public Timeline getTimeline() {
initTimeline();
return timeline;
}
private void initTimeline() {
if (timeline != null) {
return;
}
Map<Integer, VideoFrameTag> frames = new HashMap<>();
SWF.populateVideoFrames(characterID, swf.getTags(), frames);
Set<Integer> frameNums = new TreeSet<>(frames.keySet());
int maxFr = 0;
for (int f : frameNums) {
maxFr = f;
}
List<Tag> tags = new ArrayList<>();
for (int f = 0; f < maxFr; f++) {
if (frames.containsKey(f)) {
tags.add(frames.get(f));
}
tags.add(new PlaceObject2Tag(swf, false, 1, characterID, new MATRIX(), null, f, null, -1, null));
tags.add(new ShowFrameTag(swf));
}
this.tags = new ReadOnlyTagList(tags);
timeline = new Timeline(swf, this, characterID, getRect()) {
@Override
public void getSounds(int frame, int time, ButtonTag mouseOverButton, int mouseButton, List<Integer> sounds, List<String> soundClasses, List<SOUNDINFO> soundInfos) {
}
};
}
@Override
public void resetTimeline() {
timeline = null;
}
@Override
public ReadOnlyTagList getTags() {
initTimeline();
return tags;
}
@Override
public void removeTag(int index) {
}
@Override
public void removeTag(Tag tag) {
}
@Override
public void addTag(Tag tag) {
}
@Override
public void addTag(int index, Tag tag) {
}
@Override
public void replaceTag(int index, Tag newTag) {
}
@Override
public void replaceTag(Tag oldTag, Tag newTag) {
}
@Override
public int indexOfTag(Tag tag) {
return tags.indexOf(tag);
}
@Override
public void setFrameCount(int frameCount) {
}
@Override
public int getFrameCount() {
return numFrames;
}
}

View File

@@ -0,0 +1,7 @@
package com.jpexs.video;
import java.awt.image.BufferedImage;
public interface FrameListener {
public void newFrameRecieved(BufferedImage image);
}

View File

@@ -0,0 +1,334 @@
package com.jpexs.video;
import com.jpexs.decompiler.flash.configuration.Configuration;
import java.awt.image.BufferedImage;
import java.lang.annotation.Native;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import com.sun.jna.NativeLibrary;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.swing.SwingUtilities;
import uk.co.caprica.vlcj.factory.MediaPlayerFactory;
import uk.co.caprica.vlcj.factory.discovery.NativeDiscovery;
import uk.co.caprica.vlcj.factory.discovery.provider.DirectoryProviderDiscoveryStrategy;
import uk.co.caprica.vlcj.factory.discovery.strategy.NativeDiscoveryStrategy;
import uk.co.caprica.vlcj.media.MediaRef;
import uk.co.caprica.vlcj.medialist.MediaList;
import uk.co.caprica.vlcj.medialist.MediaListRef;
import uk.co.caprica.vlcj.player.base.MediaPlayer;
import uk.co.caprica.vlcj.player.base.MediaPlayerEventAdapter;
import uk.co.caprica.vlcj.player.embedded.EmbeddedMediaPlayer;
import uk.co.caprica.vlcj.player.embedded.videosurface.CallbackVideoSurface;
import uk.co.caprica.vlcj.player.embedded.videosurface.VideoSurfaceAdapters;
import uk.co.caprica.vlcj.player.embedded.videosurface.callback.BufferFormat;
import uk.co.caprica.vlcj.player.embedded.videosurface.callback.BufferFormatCallback;
import uk.co.caprica.vlcj.player.embedded.videosurface.callback.RenderCallback;
import uk.co.caprica.vlcj.player.embedded.videosurface.callback.format.RV32BufferFormat;
import uk.co.caprica.vlcj.player.list.MediaListPlayer;
import uk.co.caprica.vlcj.player.list.MediaListPlayerEventAdapter;
import uk.co.caprica.vlcj.player.list.PlaybackMode;
public class SimpleMediaPlayer {
private List<FrameListener> listeners = new ArrayList<>();
private final EmbeddedMediaPlayer embeddedMediaPlayer;
private final MediaListPlayer mediaListPlayer;
private final MediaPlayerFactory mediaPlayerFactory;
private boolean paused = true;
private long time = 0L;
private float position = 0f;
private long length = 0L;
private boolean positionSet = false;
private boolean loaded = false;
private boolean finished = false;
private boolean singleFrame = false;
private final Object displayLock = new Object();
private String file;
private MyRenderCallback callback;
public long getLength() {
return length;
}
public void addFrameListener(FrameListener listener) {
listeners.add(listener);
}
public void removeFrameListener(FrameListener listener) {
listeners.remove(listener);
}
public void play(String file) {
loaded = false;
this.file = file;
//embeddedMediaPlayer.media().play(file); //.play(file);
MediaList mediaList = mediaPlayerFactory.media().newMediaList();
mediaList.media().add(file);
MediaListRef mediaListRef = mediaList.newMediaListRef();
try {
mediaListPlayer.list().setMediaList(mediaListRef);
} finally {
mediaListRef.release();
}
mediaListPlayer.controls().play();
}
public void stop() {
embeddedMediaPlayer.controls().stop();
}
public void pause() {
if (paused) {
paused = false;
if (length != 0) {
embeddedMediaPlayer.controls().setPosition(((float) time) / length);
}
embeddedMediaPlayer.controls().play();
return;
}
embeddedMediaPlayer.controls().pause();
paused = true;
}
public void setTime(long time) {
synchronized (displayLock) {
this.time = time;
positionSet = false;
if (paused) {
singleFrame = true;
}
embeddedMediaPlayer.controls().setTime(time);
embeddedMediaPlayer.controls().play();
if (paused) {
try {
displayLock.wait();
} catch (InterruptedException ex) {
Logger.getLogger(SimpleMediaPlayer.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
//embeddedMediaPlayer.controls().play();
}
public void setPosition(float position) {
synchronized (displayLock) {
this.position = position;
positionSet = true;
singleFrame = true;
}
//System.out.println("setting position: "+ position);
embeddedMediaPlayer.controls().pause();
embeddedMediaPlayer.controls().setPosition(position);
embeddedMediaPlayer.controls().play();
/*if (paused) {
try {
displayLock.wait();
} catch (InterruptedException ex) {
Logger.getLogger(SimpleMediaPlayer.class.getName()).log(Level.SEVERE, null, ex);
}
}*/
//embeddedMediaPlayer.controls().play();
}
public boolean isPaused() {
return paused;
}
/*public void rewind() {
System.out.println("rewinding");
//embeddedMediaPlayer.controls().stop();
//embeddedMediaPlayer.controls().start();
position = 0f;
loaded = false;
embeddedMediaPlayer.media().play(file);
//embeddedMediaPlayer.controls().setPosition(0f);
//embeddedMediaPlayer.controls().play();
System.out.println("rewound");
}*/
public SimpleMediaPlayer() {
BufferFormatCallback bufferFormatCallback = new BufferFormatCallback() {
@Override
public BufferFormat getBufferFormat(int sourceWidth, int sourceHeight) {
return new RV32BufferFormat(sourceWidth, sourceHeight);
}
@Override
public void allocatedBuffers(ByteBuffer[] buffers) {
}
};
callback = new MyRenderCallback(listeners);
CallbackVideoSurface callbackVideoSurface = new CallbackVideoSurface(bufferFormatCallback, callback,
false, VideoSurfaceAdapters.getVideoSurfaceAdapter());
NativeLibrary.addSearchPath("libvlc", Configuration.vlcPlayerLocation.get());
mediaPlayerFactory = new MediaPlayerFactory();
embeddedMediaPlayer = mediaPlayerFactory.mediaPlayers().newEmbeddedMediaPlayer();
callbackVideoSurface.attach(embeddedMediaPlayer);
embeddedMediaPlayer.videoSurface().set(callbackVideoSurface);
embeddedMediaPlayer.videoSurface().attachVideoSurface();
embeddedMediaPlayer.events().addMediaPlayerEventListener(new MediaPlayerEventAdapter() {
@Override
public void lengthChanged(uk.co.caprica.vlcj.player.base.MediaPlayer mediaPlayer, long newLength) {
length = newLength;
}
@Override
public void timeChanged(uk.co.caprica.vlcj.player.base.MediaPlayer mediaPlayer, long newTime) {
SimpleMediaPlayer.this.time = newTime;
}
@Override
public void positionChanged(uk.co.caprica.vlcj.player.base.MediaPlayer mediaPlayer, float newPosition) {
SimpleMediaPlayer.this.position = newPosition;
//System.out.println("position changed to "+newPosition);
}
@Override
public void finished(MediaPlayer mediaPlayer) {
/*System.out.println("finished");
finished = true;
callback.sendImage();
new Thread() {
@Override
public void run() {
System.out.println("finished settime 0");
mediaPlayer.controls().setPosition(0f);
System.out.println("finished play");
mediaPlayer.controls().play();
System.out.println("/finished");
}
}.start();*/
}
@Override
public void stopped(MediaPlayer mediaPlayer) {
System.out.println("stopped");
}
@Override
public void playing(uk.co.caprica.vlcj.player.base.MediaPlayer mediaPlayer) {
if (!loaded) {
if (positionSet) {
embeddedMediaPlayer.controls().setPosition(position);
} else {
embeddedMediaPlayer.controls().setPosition(((float) time) / length);
}
}
//System.out.println("playing");
finished = false;
//embeddedMediaPlayer.controls().setRepeat(true);
}
});
//embeddedMediaPlayer.controls().setRepeat(true);
mediaListPlayer = mediaPlayerFactory.mediaPlayers().newMediaListPlayer();
mediaListPlayer.events().addMediaListPlayerEventListener(new MediaListPlayerEventAdapter() {
@Override
public void nextItem(MediaListPlayer mediaListPlayer, MediaRef item) {
//System.out.println("nextItem()");
}
});
mediaListPlayer.mediaPlayer().setMediaPlayer(embeddedMediaPlayer);
mediaListPlayer.controls().setMode(PlaybackMode.LOOP);
}
public boolean isFinished() {
return finished;
}
private class MyRenderCallback implements RenderCallback {
private List<FrameListener> videoSurfaces;
private int width;
private int height;
private BufferedImage image;
private int[] rgbBuffer;
public BufferedImage getImage() {
return image;
}
public void sendImage() {
for (FrameListener fl : videoSurfaces) {
fl.newFrameRecieved(this.image);
}
}
public MyRenderCallback(List<FrameListener> listeners) {
this.videoSurfaces = listeners;
}
@Override
public void display(uk.co.caprica.vlcj.player.base.MediaPlayer mediaPlayer, ByteBuffer[] nativeBuffers, BufferFormat bufferFormat) {
synchronized (displayLock) {
if (singleFrame) {
singleFrame = false;
if (image == null) {
this.width = bufferFormat.getWidth();
this.height = bufferFormat.getHeight();
this.image = new BufferedImage(this.width, this.height, BufferedImage.TYPE_INT_ARGB);
rgbBuffer = new int[image.getWidth() * image.getHeight()];
}
nativeBuffers[0].asIntBuffer().get(rgbBuffer, 0, bufferFormat.getHeight() * bufferFormat.getWidth());
image.setRGB(0, 0, image.getWidth(), image.getHeight(), rgbBuffer, 0, image.getWidth());
}
if (!loaded) {
loaded = true;
//System.out.println("just loaded");
if (positionSet) {
embeddedMediaPlayer.controls().setPosition(position);
} else {
embeddedMediaPlayer.controls().setPosition(((float) time) / length);
}
}
}
embeddedMediaPlayer.controls().pause();
sendImage();
//System.out.println("display return");
}
}
}