master
Marcus Gosselin 2015-02-14 14:11:27 -05:00
parent 50213fc661
commit 20fce00161
8 changed files with 231 additions and 341 deletions

View File

@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<classpath> <classpath>
<classpathentry kind="src" path="src"/> <classpathentry kind="src" path="src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"/> <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>
<classpathentry kind="lib" path="lib/commons-io-2.4.jar"/> <classpathentry kind="lib" path="lib/commons-io-2.4.jar"/>
<classpathentry kind="lib" path="lib/jackson-core-2.2.3.jar"/> <classpathentry kind="lib" path="lib/jackson-core-2.2.3.jar"/>
<classpathentry kind="lib" path="lib/jackson-databind-2.2.3.jar"/> <classpathentry kind="lib" path="lib/jackson-databind-2.2.3.jar"/>

View File

@ -1,11 +1,12 @@
eclipse.preferences.version=1 eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.compliance=1.7
org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.lineNumber=generate
org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate
org.eclipse.jdt.core.compiler.debug.sourceFile=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.source=1.8 org.eclipse.jdt.core.compiler.source=1.7

View File

@ -0,0 +1,2 @@
eclipse.preferences.version=1
org.eclipse.ltk.core.refactoring.enable.project.refactoring.history=false

View File

@ -1,6 +0,0 @@
import java.awt.image.BufferedImage;
public interface ImageListener {
public abstract void newImage(BufferedImage image);
}

View File

@ -1,331 +0,0 @@
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.sql.Connection;
import java.util.Iterator;
import java.util.stream.Stream;
import javax.imageio.ImageIO;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.TeeOutputStream;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
/**
* now you need to have an instance. because only one request at a time. new
* request? new instance. later, TODO, make there be a busy variable or
* something.
*
* @author Marcus
*
*/
public class ImgurRequest {
// used if no path is pre specified.
private String baseDir = System.getenv("USERPROFILE")
+ "\\Desktop\\Imgur\\";
// my personal API key so i can access imgur api and stuff
private String CLIENT_ID = "76535d44f1f94da";
// to track progress
// prediction because divide by zero errors are infectious
private volatile int totalImages = 0, imagesComplete = 0,
predictedTotal = 1;
// busy? ornahhhh
private boolean busy = false;
// have we scanned all the pages in the current request?
private boolean scannedAllPages = true;
// title so this can have a label
private String title = "";
// how many pages scanned and to be scanned
private int pagesScanned = 0, pagesToScan = 0;
private ImageListener listener;
public ImgurRequest(ImageListener listener) {
this.listener = listener;
}
// sort is usually time but i thought i'd be nice to yall.
public void saveSubreddit(final String subreddit, final int pages,
final String sort) {
totalImages = imagesComplete = 0;
scannedAllPages = false;
busy = true;
predictedTotal = pages * 60;
title = subreddit;
pagesToScan = pages;
pagesScanned = 0;
new Thread(new Runnable() {
public void run() {
// https://api.imgur.com/3/gallery/r/{subreddit}/{sort}/{page}
for (int page = 0; page < pages; page++) {
try {
String path = "https://api.imgur.com/3/gallery/r/"
+ subreddit + "/" + sort + "/" + page + ".json";
HttpURLConnection connection = (HttpURLConnection) ((new URL(
path)).openConnection());
connection.setRequestMethod("GET");
connection.addRequestProperty("Authorization",
"client-id " + CLIENT_ID);
connection.connect();
if (connection.getResponseCode() == 200) {
InputStream response = connection.getInputStream();
saveArray(response);
} else {
title = "error code "
+ connection.getResponseCode();
busy = false;
}
} catch (Exception e) {
e.printStackTrace();
busy = false;
}
pagesScanned++;
}
scannedAllPages = true;
}
}).start();
}
// sort is usually time but i thought i'd be nice to yall.
public void saveSubmitted(final String user, final int pages) {
totalImages = imagesComplete = 0;
scannedAllPages = false;
busy = true;
predictedTotal = pages * 60;
title = user;
pagesToScan = pages;
pagesScanned = 0;
new Thread(new Runnable() {
public void run() {
// https://api.imgur.com/3/gallery/r/{subreddit}/{sort}/{page}
for (int page = 0; page < pages; page++) {
try {
String path = "https://api.imgur.com/3/account/" + user
+ "/submissions/" + page + ".json";
HttpURLConnection connection = (HttpURLConnection) ((new URL(
path)).openConnection());
connection.setRequestMethod("GET");
connection.addRequestProperty("Authorization",
"client-id " + CLIENT_ID);
connection.connect();
if (connection.getResponseCode() == 200) {
InputStream response = connection.getInputStream();
saveArray(response);
} else {
title = "error code "
+ connection.getResponseCode();
busy = false;
}
} catch (Exception e) {
e.printStackTrace();
busy = false;
}
pagesScanned++;
}
scannedAllPages = true;
}
}).start();
}
public void saveArray(final InputStream response) {
new Thread(new Runnable() {
public void run() {
try {
ObjectMapper om = new ObjectMapper();
JsonNode root;
root = om.readTree(response);
JsonNode imagesNode = root.get("data");
Iterator<JsonNode> imagesIterator = imagesNode.iterator();
// count em up first
int images = 0;
while (imagesIterator.hasNext()) {
images++;
imagesIterator.next();
}
totalImages += images;
imagesIterator = imagesNode.iterator();
int imageCounter = 1;
while (imagesIterator.hasNext()) {
JsonNode item = imagesIterator.next();
boolean album = item.get("is_album").asBoolean();
if (!album) {
String id = item.get("id").asText();
int fileSize = item.get("size").asInt();
String extension = item.get("type").asText();
extension = parseExtension(extension);
String subreddit = item.get("section").asText();
saveID(id, subreddit + "\\", extension, fileSize);
} else System.out.println("Theres an album here but we dont care yet");
imagesComplete++;
imageCounter++;
}
if (imagesComplete == totalImages) {
busy = false;
}
} catch (Exception e) {
e.printStackTrace();
}
}
}).start();
}
private String parseExtension(String ext) {
if (ext.equals("image/jpeg")) {
return ".jpg";
}
if (ext.equals("image/png")) {
return ".png";
}
if (ext.equals("image/gif")) {
return ".gif";
}
// eventually fill this with all possible cases
// nvm, i think this is all cases.
return ".jpg";
}
public void saveID(String hash, String subfolder, String extension,
int filesize) {
try {
// make sure our directories exist no matter what
new File(baseDir + subfolder).mkdirs();
new File(baseDir + "backgrounds\\" + subfolder).mkdirs();
// if we haven't fully saved this yet...
if (!(new File(baseDir + subfolder + hash + extension).length() == filesize)) {
InputStream in = new URL("http://i.imgur.com/" + hash
+ extension).openConnection().getInputStream();
OutputStream fileOut = new FileOutputStream(baseDir + subfolder
+ hash + extension);
copy(in, fileOut);
}
if (listener != null) {
try {
// listener.newImage(ImageIO.read(new File(baseDir +
// subfolder
// + hash + extension)));
listener.newImage(ImageIO.read(new URL(
"http://i.imgur.com/" + hash + extension)
.openConnection().getInputStream()));
} catch (Exception e) {
// eh
}
}
} catch (Exception e) {
e.printStackTrace();
System.out.println(hash);
System.out.println(extension);
System.out.println("" + filesize);
}
}
/**
* to copy one stream thing to another stream thing. it goes sanic fast.
*
* @param input
* @param output
*/
private void copy(InputStream input, OutputStream output) {
try {
IOUtils.copy(input, output);
} catch (Exception e) {
} finally {
if (input != null) {
try {
input.close();
} catch (IOException e) {
}
}
if (output != null) {
try {
output.close();
} catch (IOException e) {
}
}
}
}
public double getProgress() {
return scannedAllPages ? (double) imagesComplete / (double) totalImages
: (double) imagesComplete / (double) predictedTotal;
}
public double getScanProgress() {
return (double) (pagesScanned) / pagesToScan;
}
public int getImagesDiscovered() {
return totalImages;
}
public int getImagesDownloaded() {
return imagesComplete;
}
public String getTitle() {
return title;
}
public boolean isBusy() {
return busy;
}
// you deserve a break
// Eugenia Suarez
// Mery del Cerro
// google them for a good time
}

View File

@ -0,0 +1,7 @@
package imgurlibrary;
import java.awt.image.BufferedImage;
public interface ImageListener {
public abstract void sendImage(BufferedImage image);
}

View File

@ -0,0 +1,95 @@
package imgurlibrary;
import java.awt.image.BufferedImage;
import java.io.File;
public abstract class ImgurRequest implements Runnable {
public static String LOCAL_ROOT = System.getenv("USERPROFILE") + "\\Desktop\\imgur";
protected final String AuthorizationToken = "76535d44f1f94da";
public final String name;
protected boolean busy;
private final ImageListener listener;
private volatile int downloaded, discovered, failed;
static {
if (!new File(LOCAL_ROOT).exists()) {
new File(LOCAL_ROOT).mkdirs();
}
}
protected synchronized final void discovered(int i) {
discovered+=i;
}
protected synchronized final void discovered() {
discovered++;
}
protected synchronized final void downloaded(int i) {
downloaded+=i;
}
protected synchronized final void downloaded() {
downloaded++;
}
protected synchronized final void failed(int i) {
failed+=i;
}
protected synchronized final void failed() {
failed++;
}
protected void sendImage(BufferedImage image) {
if (listener != null)
listener.sendImage(image);
}
protected ImgurRequest(String name, ImageListener listener) {
busy = true;
this.name = name;
discovered = 0;
downloaded = 0;
failed = 0;
this.listener = listener;
}
public final double getProgress() {
// bc divide by zero error
return ((double)(downloaded + failed) / (discovered == 0 ? 1 : discovered));
}
public abstract double getScanProgress(); // idunno how YOU want to
// calculate this but uh
public final boolean isBusy() { // i aint doing it for you so....
return busy; // ~All interfaces ever
}
public abstract void run();
protected final String getExt(String MIME) {
switch (MIME) {
case "image/jpeg":
return ".jpeg";
case "image/png":
return ".png";
case "image/gif":
return ".gif";
default:
return ".jpg";
}
}
public int getImagesDiscovered() {
return discovered;
}
public int getImagesDownloaded() {
return downloaded;
}
public String getTitle() {
return name;
}
}

View File

@ -0,0 +1,122 @@
package imgurlibrary;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Iterator;
import javax.imageio.ImageIO;
import org.apache.commons.io.IOUtils;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
public class SubredditRequest extends ImgurRequest {
private final int pages;
private int pagesScanned;
private static final String BASE_URL = "https://api.imgur.com/3/gallery/r/";
private final String URL;
private final String subreddit;
private final String LOCAL_ROOT;
public SubredditRequest(String subreddit, int pages, boolean async) {
this(subreddit, pages, async, null);
}
public SubredditRequest(String subreddit, int pages, boolean async, ImageListener listener) {
super(subreddit, listener);
this.pages = pages;
pagesScanned = 0;
busy = true;
URL = BASE_URL + subreddit + "/time/";
this.subreddit = subreddit;
LOCAL_ROOT = ImgurRequest.LOCAL_ROOT + File.separatorChar + subreddit;
if (!new File(LOCAL_ROOT).exists()) {
new File(LOCAL_ROOT).mkdirs();
}
if (async)
new Thread(this).start();
else
run();
}
@Override
public double getScanProgress() {
return pagesScanned / (double) (pages);
}
@Override
public void run() {
// && busy so we can easily exit if a fatal error happens
for (int page = 0; page < pages && busy; page++) {
try {
String path = "https://api.imgur.com/3/gallery/r/emmawatson/time/0.json";
HttpURLConnection connection = (HttpURLConnection) ((new URL(path)).openConnection());
connection.setRequestMethod("GET");
connection.addRequestProperty("Authorization", "client-id 76535d44f1f94da");
connection.connect();
if (connection.getResponseCode() == 200) {
InputStream response = connection.getInputStream();
savePage(response);
} else {
busy = false;
}
} catch (Exception e) {
e.printStackTrace(System.out);
}
}
}
private void savePage(final InputStream response) {
new Thread(new Runnable() {
public void run() {
try {
ObjectMapper om = new ObjectMapper();
JsonNode root = om.readTree(response);
JsonNode data = root.get("data");
Iterator<JsonNode> iterator = data.iterator();
while (iterator.hasNext()) {
iterator.next();
discovered();
}
pagesScanned++;
// reset the iterator ya doof
iterator = data.iterator();
while (iterator.hasNext()) {
JsonNode image = iterator.next();
// no albums here so no worries
String id = image.get("id").asText();
String ext = getExt(image.get("type").asText());
InputStream in = new URL("http://i.imgur.com/" + id + ext).openConnection().getInputStream();
OutputStream out = new FileOutputStream(new File(LOCAL_ROOT + File.separatorChar + id + ext));
IOUtils.copy(in, out);
downloaded();
sendImage(ImageIO.read(new File(LOCAL_ROOT + File.separatorChar + id + ext)));
}
} catch (Exception e) {
e.printStackTrace();
failed();
}
}
}).start();
}
public static void main(String[] args) {
new SubredditRequest("derp", 1, true);
}
}