Download utils

src/main/java/com/volmit/iris/util/network/DL.java

@@ -0,0 +1,322 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2021 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.volmit.iris.util.network;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.net.URLConnection;
+
+import com.volmit.iris.util.collection.KSet;
+import com.volmit.iris.util.io.IO;
+import com.volmit.iris.util.scheduling.ChronoLatch;
+
+public abstract class DL
+{
+	protected File d;
+	protected URL u;
+	protected ChronoLatch latch;
+	protected KSet<DownloadFlag> flags;
+	protected MeteredOutputStream o;
+	protected DownloadState state;
+	protected int timeout;
+	protected long size;
+	protected long start;
+	protected long downloaded;
+	protected long currentChunk;
+	protected long lastChunk;
+	protected long bps;
+	protected int bufferSize;
+	protected long lastPull;
+	protected DownloadMonitor m;
+	
+	public DL(URL u, File d, DownloadFlag...downloadFlags)
+	{
+		this.d = d;
+		this.u = u;
+		size = -1;
+		lastPull = -1;
+		downloaded = 0;
+		bufferSize = 8192 * 32;
+		currentChunk = 0;
+		lastChunk = -1;
+		bps = -1;
+		start = -1;
+		timeout = 10000;
+		state = DownloadState.NEW;
+		flags = new KSet<>();
+		latch = new ChronoLatch(500);
+
+		for(DownloadFlag i : downloadFlags)
+		{
+			flags.add(i);
+		}
+	}
+	
+	public void monitor(DownloadMonitor m)
+	{
+		this.m = m;
+	}
+	
+	public void update()
+	{
+		if(m != null)
+		{
+			m.onUpdate(state, getProgress(), getElapsed(), getTimeLeft(), bps, getDiskBytesPerSecond(), size, downloaded, bufferSize, getBufferUse());
+		}
+	}
+	
+	public boolean hasFlag(DownloadFlag f)
+	{
+		return flags.contains(f);
+	}
+	
+	public boolean isState(DownloadState s)
+	{
+		return state.equals(s);
+	}
+	
+	protected void state(DownloadState s)
+	{
+		this.state = s;
+		update();
+	}
+	
+	public int getBufferSize()
+	{
+		return bufferSize;
+	}
+	
+	public void start() throws IOException
+	{
+		if(!isState(DownloadState.NEW))
+		{
+			throw new DownloadException("Cannot start download while " + state.toString());
+		}
+		
+		state(DownloadState.STARTING);
+		
+		if(hasFlag(DownloadFlag.CALCULATE_SIZE))
+		{
+			size = calculateSize();
+		}
+		
+		start = System.currentTimeMillis();
+		downloaded = 0;
+		bps = 0;
+		lastChunk = System.currentTimeMillis();
+		o = new MeteredOutputStream(new FileOutputStream(d), 100);
+		openStream();
+		state(DownloadState.DOWNLOADING);
+	}
+	
+	protected abstract long download() throws IOException;
+	
+	protected abstract void openStream() throws IOException;
+	
+	protected abstract void closeStream() throws IOException;
+	
+	public void downloadChunk() throws IOException
+	{
+		if(!isState(DownloadState.DOWNLOADING))
+		{
+			throw new DownloadException("Cannot download while " + state.toString());
+		}
+		
+		long d = download();
+		lastPull = d;
+		
+		if(d < 0)
+		{
+			finishDownload();
+			return;
+		}
+		
+		downloaded += d;
+		currentChunk += d;
+		
+		double chunkTime = (double)(System.currentTimeMillis() - lastChunk) / 1000D;
+		bps = (long) ((double)currentChunk / chunkTime);
+		
+		if(latch.flip())
+		{
+			update();
+		}
+	}
+	
+	public double getBufferUse()
+	{
+		return (double)lastPull / (double)bufferSize;
+	}
+	
+	private void finishDownload() throws IOException 
+	{
+		if(!isState(DownloadState.NEW))
+		{
+			throw new DownloadException("Cannot finish download while " + state.toString());
+		}
+		
+		closeStream();
+		o.close();
+		state(DownloadState.COMPLETE);
+	}
+
+	public long getElapsed()
+	{
+		return System.currentTimeMillis() - start;
+	}
+	
+	public long getRemaining()
+	{
+		return size - downloaded;
+	}
+	
+	public long getTimeLeft()
+	{
+		return (long) (((double)getRemaining() / (double)bps) * 1000D);
+	}
+	
+	public long getDiskBytesPerSecond()
+	{
+		if(o == null)
+		{
+			return -1;
+		}
+		
+		return o.getBps();
+	}
+	
+	public long getBytesPerSecond()
+	{
+		return bps;
+	}
+	
+	public double getProgress()
+	{
+		return hasProgress() ? ((double)downloaded / (double)size) : -1D;
+	}
+	
+	public boolean hasProgress()
+	{
+		return size > 0;
+	}
+
+	private long calculateSize() throws IOException
+	{
+		URLConnection c = u.openConnection();
+		c.setConnectTimeout((int) timeout);
+		c.setReadTimeout((int) timeout);
+		c.connect();
+		return c.getContentLengthLong();
+	}
+	
+	public enum DownloadFlag
+	{
+		CALCULATE_SIZE
+	}
+	
+	public enum DownloadState
+	{
+		NEW,
+		STARTING,
+		DOWNLOADING,
+		FINALIZING,
+		COMPLETE,
+		FAILED
+	}
+	
+	public static class ThrottledDownload extends Download
+	{
+		private long mbps;
+		
+		public ThrottledDownload(URL u, File d, long mbps, DownloadFlag... downloadFlags) {
+			super(u, d, downloadFlags);
+			this.mbps = mbps;
+		}
+		
+		@Override
+		protected long download() throws IOException
+		{
+			if(getBytesPerSecond() > mbps)
+			{
+				try 
+				{
+					Thread.sleep(40);
+				} 
+				
+				catch (InterruptedException e) 
+				{
+					e.printStackTrace();
+				}
+				
+				return IO.transfer(in, o, 8192, mbps/20);
+			}
+			
+			return IO.transfer(in, o, 8192, bufferSize);
+		}
+	}
+	
+	public static class DoubleBufferedDownload extends Download
+	{
+		protected BufferedOutputStream os;
+		
+		public DoubleBufferedDownload(URL u, File d, DownloadFlag... downloadFlags)
+		{
+			super(u, d, downloadFlags);
+		}
+		
+		@Override
+		protected void openStream() throws IOException 
+		{
+			os = new BufferedOutputStream(o, 8192*16);
+			in = new BufferedInputStream(u.openStream(), 8192*16);
+			buf = new byte[8192 * 2];
+		}
+	}
+	
+	public static class Download extends DL
+	{
+		protected InputStream in;
+		protected byte[] buf;
+		public Download(URL u, File d, DownloadFlag... downloadFlags) {
+			super(u, d, downloadFlags);
+		}
+
+		@Override
+		protected long download() throws IOException 
+		{
+			return IO.transfer(in, o, buf, bufferSize);
+		}
+
+		@Override
+		protected void openStream() throws IOException {
+			in = u.openStream();
+			buf = new byte[8192];
+		}
+
+		@Override
+		protected void closeStream() throws IOException {
+			in.close();
+		}
+	}
+}

src/main/java/com/volmit/iris/util/network/DownloadException.java

@@ -0,0 +1,42 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2021 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.volmit.iris.util.network;
+
+import java.io.IOException;
+
+public class DownloadException extends IOException
+{
+	private static final long serialVersionUID = 5137918663903349839L;
+
+	public DownloadException() {
+		super();
+	}
+
+	public DownloadException(String message, Throwable cause) {
+		super(message, cause);
+	}
+
+	public DownloadException(String message) {
+		super(message);
+	}
+
+	public DownloadException(Throwable cause) {
+		super(cause);
+	}
+}

src/main/java/com/volmit/iris/util/network/DownloadMonitor.java

@@ -0,0 +1,25 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2021 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.volmit.iris.util.network;
+
+@FunctionalInterface
+public interface DownloadMonitor 
+{
+	public void onUpdate(DL.DownloadState state, double progress, long elapsed, long estimated, long bps, long iobps, long size, long downloaded, long buffer, double bufferuse);
+}

src/main/java/com/volmit/iris/util/network/MeteredInputStream.java

@@ -0,0 +1,120 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2021 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.volmit.iris.util.network;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class MeteredInputStream extends InputStream
+{
+	private InputStream os;
+	private long written;
+	private long totalWritten;
+	private long since;
+	private boolean auto;
+	private long interval;
+	private long bps;
+	
+	public MeteredInputStream(InputStream os, long interval)
+	{
+		this.os = os;
+		written = 0;
+		totalWritten = 0;
+		auto = true;
+		this.interval = interval;
+		bps = 0;
+		since = System.currentTimeMillis();
+	}
+	
+	public MeteredInputStream(InputStream os)
+	{
+		this(os, 100);
+		auto = false;
+	}
+
+	@Override
+	public int read() throws IOException
+	{
+		written++;
+		totalWritten++;
+		
+		if(auto && System.currentTimeMillis() - getSince() > interval)
+		{
+			pollRead();
+		}
+		
+		return os.read();
+	}
+	
+	public long getSince()
+	{
+		return since;
+	}
+	
+	public long getRead()
+	{
+		return written;
+	}
+	
+	public long pollRead()
+	{
+		long w = written;
+		written = 0;
+		double secondsElapsedSince = (double) (System.currentTimeMillis() - since) / 1000.0;
+		bps = (long) ((double) w / secondsElapsedSince);
+		since = System.currentTimeMillis();
+		
+		return w;
+	}
+
+	public void close() throws IOException
+	{
+		os.close();
+	}
+
+	public boolean isAuto()
+	{
+		return auto;
+	}
+
+	public void setAuto(boolean auto) 
+	{
+		this.auto = auto;
+	}
+
+	public long getInterval()
+	{
+		return interval;
+	}
+
+	public void setInterval(long interval)
+	{
+		this.interval = interval;
+	}
+
+	public long getTotalRead() 
+	{
+		return totalWritten;
+	}
+
+	public long getBps() 
+	{
+		return bps;
+	}
+}

src/main/java/com/volmit/iris/util/network/MeteredOutputStream.java

@@ -0,0 +1,118 @@
+/*
+ * Iris is a World Generator for Minecraft Bukkit Servers
+ * Copyright (c) 2021 Arcane Arts (Volmit Software)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.volmit.iris.util.network;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class MeteredOutputStream extends OutputStream
+{
+	private OutputStream os;
+	private long written;
+	private long totalWritten;
+	private long since;
+	private boolean auto;
+	private long interval;
+	private long bps;
+	
+	public MeteredOutputStream(OutputStream os, long interval)
+	{
+		this.os = os;
+		written = 0;
+		totalWritten = 0;
+		auto = true;
+		this.interval = interval;
+		bps = 0;
+		since = System.currentTimeMillis();
+	}
+	
+	public MeteredOutputStream(OutputStream os)
+	{
+		this(os, 100);
+		auto = false;
+	}
+
+	@Override
+	public void write(int b) throws IOException 
+	{
+		os.write(b);
+		written++;
+		totalWritten++;
+		
+		if(auto && System.currentTimeMillis() - getSince() > interval)
+		{
+			pollWritten();
+		}
+	}
+	
+	public long getSince()
+	{
+		return since;
+	}
+	
+	public long getWritten()
+	{
+		return written;
+	}
+	
+	public long pollWritten()
+	{
+		long w = written;
+		written = 0;
+		double secondsElapsedSince = (double) (System.currentTimeMillis() - since) / 1000.0;
+		bps = (long) ((double) w / secondsElapsedSince);
+		since = System.currentTimeMillis();
+		return w;
+	}
+
+	public void close() throws IOException
+	{
+		os.close();
+	}
+
+	public boolean isAuto()
+	{
+		return auto;
+	}
+
+	public void setAuto(boolean auto) 
+	{
+		this.auto = auto;
+	}
+
+	public long getInterval()
+	{
+		return interval;
+	}
+
+	public void setInterval(long interval)
+	{
+		this.interval = interval;
+	}
+
+	public long getTotalWritten() 
+	{
+		return totalWritten;
+	}
+
+	public long getBps() 
+	{
+		return bps;
+	}
+}