From 20fce00161acb6acf9e04baba3c443c0726f3bb0 Mon Sep 17 00:00:00 2001 From: Marcus Gosselin Date: Sat, 14 Feb 2015 14:11:27 -0500 Subject: [PATCH] idunno --- .classpath | 2 +- .settings/org.eclipse.jdt.core.prefs | 7 +- .../org.eclipse.ltk.core.refactoring.prefs | 2 + src/ImageListener.java | 6 - src/ImgurRequest.java | 331 ------------------ src/imgurlibrary/ImageListener.java | 7 + src/imgurlibrary/ImgurRequest.java | 95 +++++ src/imgurlibrary/SubredditRequest.java | 122 +++++++ 8 files changed, 231 insertions(+), 341 deletions(-) create mode 100644 .settings/org.eclipse.ltk.core.refactoring.prefs delete mode 100644 src/ImageListener.java delete mode 100644 src/ImgurRequest.java create mode 100644 src/imgurlibrary/ImageListener.java create mode 100644 src/imgurlibrary/ImgurRequest.java create mode 100644 src/imgurlibrary/SubredditRequest.java diff --git a/.classpath b/.classpath index e418cf1..475d7c0 100644 --- a/.classpath +++ b/.classpath @@ -1,7 +1,7 @@ - + diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 3a21537..d17b672 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -1,11 +1,12 @@ eclipse.preferences.version=1 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.compliance=1.7 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -org.eclipse.jdt.core.compiler.source=1.8 +org.eclipse.jdt.core.compiler.source=1.7 diff --git a/.settings/org.eclipse.ltk.core.refactoring.prefs b/.settings/org.eclipse.ltk.core.refactoring.prefs new file mode 100644 index 0000000..b196c64 --- /dev/null +++ b/.settings/org.eclipse.ltk.core.refactoring.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.ltk.core.refactoring.enable.project.refactoring.history=false diff --git a/src/ImageListener.java b/src/ImageListener.java deleted file mode 100644 index e9a1d71..0000000 --- a/src/ImageListener.java +++ /dev/null @@ -1,6 +0,0 @@ -import java.awt.image.BufferedImage; - - -public interface ImageListener { - public abstract void newImage(BufferedImage image); -} diff --git a/src/ImgurRequest.java b/src/ImgurRequest.java deleted file mode 100644 index 286cf0a..0000000 --- a/src/ImgurRequest.java +++ /dev/null @@ -1,331 +0,0 @@ -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.sql.Connection; -import java.util.Iterator; -import java.util.stream.Stream; - -import javax.imageio.ImageIO; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.output.TeeOutputStream; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - -/** - * now you need to have an instance. because only one request at a time. new - * request? new instance. later, TODO, make there be a busy variable or - * something. - * - * @author Marcus - * - */ -public class ImgurRequest { - - // used if no path is pre specified. - private String baseDir = System.getenv("USERPROFILE") - + "\\Desktop\\Imgur\\"; - - // my personal API key so i can access imgur api and stuff - private String CLIENT_ID = "76535d44f1f94da"; - - // to track progress - // prediction because divide by zero errors are infectious - private volatile int totalImages = 0, imagesComplete = 0, - predictedTotal = 1; - - // busy? ornahhhh - private boolean busy = false; - - // have we scanned all the pages in the current request? - private boolean scannedAllPages = true; - - // title so this can have a label - private String title = ""; - - // how many pages scanned and to be scanned - private int pagesScanned = 0, pagesToScan = 0; - - private ImageListener listener; - - public ImgurRequest(ImageListener listener) { - this.listener = listener; - } - - // sort is usually time but i thought i'd be nice to yall. - public void saveSubreddit(final String subreddit, final int pages, - final String sort) { - totalImages = imagesComplete = 0; - scannedAllPages = false; - busy = true; - predictedTotal = pages * 60; - title = subreddit; - pagesToScan = pages; - pagesScanned = 0; - new Thread(new Runnable() { - public void run() { - - // https://api.imgur.com/3/gallery/r/{subreddit}/{sort}/{page} - for (int page = 0; page < pages; page++) { - try { - String path = "https://api.imgur.com/3/gallery/r/" - + subreddit + "/" + sort + "/" + page + ".json"; - - HttpURLConnection connection = (HttpURLConnection) ((new URL( - path)).openConnection()); - - connection.setRequestMethod("GET"); - connection.addRequestProperty("Authorization", - "client-id " + CLIENT_ID); - connection.connect(); - - if (connection.getResponseCode() == 200) { - - InputStream response = connection.getInputStream(); - saveArray(response); - - } else { - title = "error code " - + connection.getResponseCode(); - busy = false; - } - - } catch (Exception e) { - e.printStackTrace(); - busy = false; - } - pagesScanned++; - } - - scannedAllPages = true; - } - }).start(); - - } - - // sort is usually time but i thought i'd be nice to yall. - public void saveSubmitted(final String user, final int pages) { - totalImages = imagesComplete = 0; - scannedAllPages = false; - busy = true; - predictedTotal = pages * 60; - title = user; - pagesToScan = pages; - pagesScanned = 0; - new Thread(new Runnable() { - public void run() { - - // https://api.imgur.com/3/gallery/r/{subreddit}/{sort}/{page} - for (int page = 0; page < pages; page++) { - try { - String path = "https://api.imgur.com/3/account/" + user - + "/submissions/" + page + ".json"; - - HttpURLConnection connection = (HttpURLConnection) ((new URL( - path)).openConnection()); - - connection.setRequestMethod("GET"); - connection.addRequestProperty("Authorization", - "client-id " + CLIENT_ID); - connection.connect(); - - if (connection.getResponseCode() == 200) { - - InputStream response = connection.getInputStream(); - - saveArray(response); - - } else { - title = "error code " - + connection.getResponseCode(); - busy = false; - } - - } catch (Exception e) { - e.printStackTrace(); - busy = false; - } - pagesScanned++; - } - - scannedAllPages = true; - } - }).start(); - - } - - public void saveArray(final InputStream response) { - new Thread(new Runnable() { - public void run() { - try { - ObjectMapper om = new ObjectMapper(); - - JsonNode root; - - root = om.readTree(response); - - JsonNode imagesNode = root.get("data"); - Iterator imagesIterator = imagesNode.iterator(); - - // count em up first - int images = 0; - while (imagesIterator.hasNext()) { - images++; - imagesIterator.next(); - } - totalImages += images; - imagesIterator = imagesNode.iterator(); - - int imageCounter = 1; - while (imagesIterator.hasNext()) { - - JsonNode item = imagesIterator.next(); - - boolean album = item.get("is_album").asBoolean(); - - if (!album) { - String id = item.get("id").asText(); - int fileSize = item.get("size").asInt(); - - String extension = item.get("type").asText(); - extension = parseExtension(extension); - - String subreddit = item.get("section").asText(); - - saveID(id, subreddit + "\\", extension, fileSize); - } else System.out.println("Theres an album here but we dont care yet"); - - imagesComplete++; - - imageCounter++; - } - - if (imagesComplete == totalImages) { - busy = false; - } - - } catch (Exception e) { - e.printStackTrace(); - } - } - }).start(); - } - - private String parseExtension(String ext) { - if (ext.equals("image/jpeg")) { - return ".jpg"; - } - if (ext.equals("image/png")) { - return ".png"; - } - if (ext.equals("image/gif")) { - return ".gif"; - } - // eventually fill this with all possible cases - // nvm, i think this is all cases. - return ".jpg"; - } - - public void saveID(String hash, String subfolder, String extension, - int filesize) { - try { - - // make sure our directories exist no matter what - new File(baseDir + subfolder).mkdirs(); - new File(baseDir + "backgrounds\\" + subfolder).mkdirs(); - - // if we haven't fully saved this yet... - if (!(new File(baseDir + subfolder + hash + extension).length() == filesize)) { - InputStream in = new URL("http://i.imgur.com/" + hash - + extension).openConnection().getInputStream(); - OutputStream fileOut = new FileOutputStream(baseDir + subfolder - + hash + extension); - - copy(in, fileOut); - } - - if (listener != null) { - try { - - // listener.newImage(ImageIO.read(new File(baseDir + - // subfolder - // + hash + extension))); - - listener.newImage(ImageIO.read(new URL( - "http://i.imgur.com/" + hash + extension) - .openConnection().getInputStream())); - - } catch (Exception e) { - // eh - } - } - } catch (Exception e) { - e.printStackTrace(); - System.out.println(hash); - System.out.println(extension); - System.out.println("" + filesize); - } - } - - /** - * to copy one stream thing to another stream thing. it goes sanic fast. - * - * @param input - * @param output - */ - private void copy(InputStream input, OutputStream output) { - try { - IOUtils.copy(input, output); - } catch (Exception e) { - } finally { - if (input != null) { - try { - input.close(); - } catch (IOException e) { - } - } - if (output != null) { - try { - output.close(); - } catch (IOException e) { - } - } - } - } - - public double getProgress() { - return scannedAllPages ? (double) imagesComplete / (double) totalImages - : (double) imagesComplete / (double) predictedTotal; - } - - public double getScanProgress() { - return (double) (pagesScanned) / pagesToScan; - } - - public int getImagesDiscovered() { - return totalImages; - } - - public int getImagesDownloaded() { - return imagesComplete; - } - - public String getTitle() { - return title; - } - - public boolean isBusy() { - return busy; - } - - // you deserve a break - // Eugenia Suarez - // Mery del Cerro - // google them for a good time -} diff --git a/src/imgurlibrary/ImageListener.java b/src/imgurlibrary/ImageListener.java new file mode 100644 index 0000000..59f4be2 --- /dev/null +++ b/src/imgurlibrary/ImageListener.java @@ -0,0 +1,7 @@ +package imgurlibrary; +import java.awt.image.BufferedImage; + + +public interface ImageListener { + public abstract void sendImage(BufferedImage image); +} diff --git a/src/imgurlibrary/ImgurRequest.java b/src/imgurlibrary/ImgurRequest.java new file mode 100644 index 0000000..6c43bcc --- /dev/null +++ b/src/imgurlibrary/ImgurRequest.java @@ -0,0 +1,95 @@ +package imgurlibrary; + +import java.awt.image.BufferedImage; +import java.io.File; + +public abstract class ImgurRequest implements Runnable { + public static String LOCAL_ROOT = System.getenv("USERPROFILE") + "\\Desktop\\imgur"; + protected final String AuthorizationToken = "76535d44f1f94da"; + public final String name; + protected boolean busy; + private final ImageListener listener; + private volatile int downloaded, discovered, failed; + static { + if (!new File(LOCAL_ROOT).exists()) { + new File(LOCAL_ROOT).mkdirs(); + } + } + + protected synchronized final void discovered(int i) { + discovered+=i; + } + + protected synchronized final void discovered() { + discovered++; + } + + protected synchronized final void downloaded(int i) { + downloaded+=i; + } + + protected synchronized final void downloaded() { + downloaded++; + } + + protected synchronized final void failed(int i) { + failed+=i; + } + + protected synchronized final void failed() { + failed++; + } + + protected void sendImage(BufferedImage image) { + if (listener != null) + listener.sendImage(image); + } + + protected ImgurRequest(String name, ImageListener listener) { + busy = true; + this.name = name; + discovered = 0; + downloaded = 0; + failed = 0; + this.listener = listener; + } + + public final double getProgress() { + // bc divide by zero error + return ((double)(downloaded + failed) / (discovered == 0 ? 1 : discovered)); + } + + public abstract double getScanProgress(); // idunno how YOU want to + // calculate this but uh + + public final boolean isBusy() { // i aint doing it for you so.... + return busy; // ~All interfaces ever + } + + public abstract void run(); + + protected final String getExt(String MIME) { + switch (MIME) { + case "image/jpeg": + return ".jpeg"; + case "image/png": + return ".png"; + case "image/gif": + return ".gif"; + default: + return ".jpg"; + } + } + + public int getImagesDiscovered() { + return discovered; + } + + public int getImagesDownloaded() { + return downloaded; + } + + public String getTitle() { + return name; + } +} diff --git a/src/imgurlibrary/SubredditRequest.java b/src/imgurlibrary/SubredditRequest.java new file mode 100644 index 0000000..c11a414 --- /dev/null +++ b/src/imgurlibrary/SubredditRequest.java @@ -0,0 +1,122 @@ +package imgurlibrary; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Iterator; + +import javax.imageio.ImageIO; + +import org.apache.commons.io.IOUtils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class SubredditRequest extends ImgurRequest { + private final int pages; + private int pagesScanned; + private static final String BASE_URL = "https://api.imgur.com/3/gallery/r/"; + private final String URL; + private final String subreddit; + private final String LOCAL_ROOT; + + public SubredditRequest(String subreddit, int pages, boolean async) { + this(subreddit, pages, async, null); + } + + public SubredditRequest(String subreddit, int pages, boolean async, ImageListener listener) { + super(subreddit, listener); + this.pages = pages; + pagesScanned = 0; + busy = true; + URL = BASE_URL + subreddit + "/time/"; + this.subreddit = subreddit; + LOCAL_ROOT = ImgurRequest.LOCAL_ROOT + File.separatorChar + subreddit; + if (!new File(LOCAL_ROOT).exists()) { + new File(LOCAL_ROOT).mkdirs(); + } + if (async) + new Thread(this).start(); + else + run(); + } + + @Override + public double getScanProgress() { + return pagesScanned / (double) (pages); + } + + @Override + public void run() { + // && busy so we can easily exit if a fatal error happens + for (int page = 0; page < pages && busy; page++) { + + try { + + String path = "https://api.imgur.com/3/gallery/r/emmawatson/time/0.json"; + + HttpURLConnection connection = (HttpURLConnection) ((new URL(path)).openConnection()); + + connection.setRequestMethod("GET"); + connection.addRequestProperty("Authorization", "client-id 76535d44f1f94da"); + connection.connect(); + + if (connection.getResponseCode() == 200) { + + InputStream response = connection.getInputStream(); + + savePage(response); + + } else { + busy = false; + } + } catch (Exception e) { + e.printStackTrace(System.out); + } + } + } + + private void savePage(final InputStream response) { + new Thread(new Runnable() { + public void run() { + try { + ObjectMapper om = new ObjectMapper(); + JsonNode root = om.readTree(response); + JsonNode data = root.get("data"); + Iterator iterator = data.iterator(); + + while (iterator.hasNext()) { + iterator.next(); + discovered(); + } + pagesScanned++; + + // reset the iterator ya doof + iterator = data.iterator(); + + while (iterator.hasNext()) { + JsonNode image = iterator.next(); + // no albums here so no worries + String id = image.get("id").asText(); + String ext = getExt(image.get("type").asText()); + InputStream in = new URL("http://i.imgur.com/" + id + ext).openConnection().getInputStream(); + OutputStream out = new FileOutputStream(new File(LOCAL_ROOT + File.separatorChar + id + ext)); + IOUtils.copy(in, out); + downloaded(); + sendImage(ImageIO.read(new File(LOCAL_ROOT + File.separatorChar + id + ext))); + } + } catch (Exception e) { + e.printStackTrace(); + failed(); + } + } + }).start(); + } + + public static void main(String[] args) { + new SubredditRequest("derp", 1, true); + } +} \ No newline at end of file