idunno
Marcus Gosselin
parent
50213fc661
commit
20fce00161
|
|
@ -1,7 +1,7 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="src" path="src"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"/>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>
|
||||
<classpathentry kind="lib" path="lib/commons-io-2.4.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jackson-core-2.2.3.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jackson-databind-2.2.3.jar"/>
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
eclipse.preferences.version=1
|
||||
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
|
||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
|
||||
org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate
|
||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
|
||||
org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
|
||||
org.eclipse.jdt.core.compiler.compliance=1.8
|
||||
org.eclipse.jdt.core.compiler.compliance=1.7
|
||||
org.eclipse.jdt.core.compiler.debug.lineNumber=generate
|
||||
org.eclipse.jdt.core.compiler.debug.localVariable=generate
|
||||
org.eclipse.jdt.core.compiler.debug.sourceFile=generate
|
||||
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
|
||||
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
|
||||
org.eclipse.jdt.core.compiler.source=1.8
|
||||
org.eclipse.jdt.core.compiler.source=1.7
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
eclipse.preferences.version=1
|
||||
org.eclipse.ltk.core.refactoring.enable.project.refactoring.history=false
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
import java.awt.image.BufferedImage;
|
||||
|
||||
|
||||
public interface ImageListener {
|
||||
public abstract void newImage(BufferedImage image);
|
||||
}
|
||||
|
|
@ -1,331 +0,0 @@
|
|||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.sql.Connection;
|
||||
import java.util.Iterator;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.commons.io.output.TeeOutputStream;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* now you need to have an instance. because only one request at a time. new
|
||||
* request? new instance. later, TODO, make there be a busy variable or
|
||||
* something.
|
||||
*
|
||||
* @author Marcus
|
||||
*
|
||||
*/
|
||||
public class ImgurRequest {
|
||||
|
||||
// used if no path is pre specified.
|
||||
private String baseDir = System.getenv("USERPROFILE")
|
||||
+ "\\Desktop\\Imgur\\";
|
||||
|
||||
// my personal API key so i can access imgur api and stuff
|
||||
private String CLIENT_ID = "76535d44f1f94da";
|
||||
|
||||
// to track progress
|
||||
// prediction because divide by zero errors are infectious
|
||||
private volatile int totalImages = 0, imagesComplete = 0,
|
||||
predictedTotal = 1;
|
||||
|
||||
// busy? ornahhhh
|
||||
private boolean busy = false;
|
||||
|
||||
// have we scanned all the pages in the current request?
|
||||
private boolean scannedAllPages = true;
|
||||
|
||||
// title so this can have a label
|
||||
private String title = "";
|
||||
|
||||
// how many pages scanned and to be scanned
|
||||
private int pagesScanned = 0, pagesToScan = 0;
|
||||
|
||||
private ImageListener listener;
|
||||
|
||||
public ImgurRequest(ImageListener listener) {
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
// sort is usually time but i thought i'd be nice to yall.
|
||||
public void saveSubreddit(final String subreddit, final int pages,
|
||||
final String sort) {
|
||||
totalImages = imagesComplete = 0;
|
||||
scannedAllPages = false;
|
||||
busy = true;
|
||||
predictedTotal = pages * 60;
|
||||
title = subreddit;
|
||||
pagesToScan = pages;
|
||||
pagesScanned = 0;
|
||||
new Thread(new Runnable() {
|
||||
public void run() {
|
||||
|
||||
// https://api.imgur.com/3/gallery/r/{subreddit}/{sort}/{page}
|
||||
for (int page = 0; page < pages; page++) {
|
||||
try {
|
||||
String path = "https://api.imgur.com/3/gallery/r/"
|
||||
+ subreddit + "/" + sort + "/" + page + ".json";
|
||||
|
||||
HttpURLConnection connection = (HttpURLConnection) ((new URL(
|
||||
path)).openConnection());
|
||||
|
||||
connection.setRequestMethod("GET");
|
||||
connection.addRequestProperty("Authorization",
|
||||
"client-id " + CLIENT_ID);
|
||||
connection.connect();
|
||||
|
||||
if (connection.getResponseCode() == 200) {
|
||||
|
||||
InputStream response = connection.getInputStream();
|
||||
saveArray(response);
|
||||
|
||||
} else {
|
||||
title = "error code "
|
||||
+ connection.getResponseCode();
|
||||
busy = false;
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
busy = false;
|
||||
}
|
||||
pagesScanned++;
|
||||
}
|
||||
|
||||
scannedAllPages = true;
|
||||
}
|
||||
}).start();
|
||||
|
||||
}
|
||||
|
||||
// sort is usually time but i thought i'd be nice to yall.
|
||||
public void saveSubmitted(final String user, final int pages) {
|
||||
totalImages = imagesComplete = 0;
|
||||
scannedAllPages = false;
|
||||
busy = true;
|
||||
predictedTotal = pages * 60;
|
||||
title = user;
|
||||
pagesToScan = pages;
|
||||
pagesScanned = 0;
|
||||
new Thread(new Runnable() {
|
||||
public void run() {
|
||||
|
||||
// https://api.imgur.com/3/gallery/r/{subreddit}/{sort}/{page}
|
||||
for (int page = 0; page < pages; page++) {
|
||||
try {
|
||||
String path = "https://api.imgur.com/3/account/" + user
|
||||
+ "/submissions/" + page + ".json";
|
||||
|
||||
HttpURLConnection connection = (HttpURLConnection) ((new URL(
|
||||
path)).openConnection());
|
||||
|
||||
connection.setRequestMethod("GET");
|
||||
connection.addRequestProperty("Authorization",
|
||||
"client-id " + CLIENT_ID);
|
||||
connection.connect();
|
||||
|
||||
if (connection.getResponseCode() == 200) {
|
||||
|
||||
InputStream response = connection.getInputStream();
|
||||
|
||||
saveArray(response);
|
||||
|
||||
} else {
|
||||
title = "error code "
|
||||
+ connection.getResponseCode();
|
||||
busy = false;
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
busy = false;
|
||||
}
|
||||
pagesScanned++;
|
||||
}
|
||||
|
||||
scannedAllPages = true;
|
||||
}
|
||||
}).start();
|
||||
|
||||
}
|
||||
|
||||
public void saveArray(final InputStream response) {
|
||||
new Thread(new Runnable() {
|
||||
public void run() {
|
||||
try {
|
||||
ObjectMapper om = new ObjectMapper();
|
||||
|
||||
JsonNode root;
|
||||
|
||||
root = om.readTree(response);
|
||||
|
||||
JsonNode imagesNode = root.get("data");
|
||||
Iterator<JsonNode> imagesIterator = imagesNode.iterator();
|
||||
|
||||
// count em up first
|
||||
int images = 0;
|
||||
while (imagesIterator.hasNext()) {
|
||||
images++;
|
||||
imagesIterator.next();
|
||||
}
|
||||
totalImages += images;
|
||||
imagesIterator = imagesNode.iterator();
|
||||
|
||||
int imageCounter = 1;
|
||||
while (imagesIterator.hasNext()) {
|
||||
|
||||
JsonNode item = imagesIterator.next();
|
||||
|
||||
boolean album = item.get("is_album").asBoolean();
|
||||
|
||||
if (!album) {
|
||||
String id = item.get("id").asText();
|
||||
int fileSize = item.get("size").asInt();
|
||||
|
||||
String extension = item.get("type").asText();
|
||||
extension = parseExtension(extension);
|
||||
|
||||
String subreddit = item.get("section").asText();
|
||||
|
||||
saveID(id, subreddit + "\\", extension, fileSize);
|
||||
} else System.out.println("Theres an album here but we dont care yet");
|
||||
|
||||
imagesComplete++;
|
||||
|
||||
imageCounter++;
|
||||
}
|
||||
|
||||
if (imagesComplete == totalImages) {
|
||||
busy = false;
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}).start();
|
||||
}
|
||||
|
||||
private String parseExtension(String ext) {
|
||||
if (ext.equals("image/jpeg")) {
|
||||
return ".jpg";
|
||||
}
|
||||
if (ext.equals("image/png")) {
|
||||
return ".png";
|
||||
}
|
||||
if (ext.equals("image/gif")) {
|
||||
return ".gif";
|
||||
}
|
||||
// eventually fill this with all possible cases
|
||||
// nvm, i think this is all cases.
|
||||
return ".jpg";
|
||||
}
|
||||
|
||||
public void saveID(String hash, String subfolder, String extension,
|
||||
int filesize) {
|
||||
try {
|
||||
|
||||
// make sure our directories exist no matter what
|
||||
new File(baseDir + subfolder).mkdirs();
|
||||
new File(baseDir + "backgrounds\\" + subfolder).mkdirs();
|
||||
|
||||
// if we haven't fully saved this yet...
|
||||
if (!(new File(baseDir + subfolder + hash + extension).length() == filesize)) {
|
||||
InputStream in = new URL("http://i.imgur.com/" + hash
|
||||
+ extension).openConnection().getInputStream();
|
||||
OutputStream fileOut = new FileOutputStream(baseDir + subfolder
|
||||
+ hash + extension);
|
||||
|
||||
copy(in, fileOut);
|
||||
}
|
||||
|
||||
if (listener != null) {
|
||||
try {
|
||||
|
||||
// listener.newImage(ImageIO.read(new File(baseDir +
|
||||
// subfolder
|
||||
// + hash + extension)));
|
||||
|
||||
listener.newImage(ImageIO.read(new URL(
|
||||
"http://i.imgur.com/" + hash + extension)
|
||||
.openConnection().getInputStream()));
|
||||
|
||||
} catch (Exception e) {
|
||||
// eh
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
System.out.println(hash);
|
||||
System.out.println(extension);
|
||||
System.out.println("" + filesize);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* to copy one stream thing to another stream thing. it goes sanic fast.
|
||||
*
|
||||
* @param input
|
||||
* @param output
|
||||
*/
|
||||
private void copy(InputStream input, OutputStream output) {
|
||||
try {
|
||||
IOUtils.copy(input, output);
|
||||
} catch (Exception e) {
|
||||
} finally {
|
||||
if (input != null) {
|
||||
try {
|
||||
input.close();
|
||||
} catch (IOException e) {
|
||||
}
|
||||
}
|
||||
if (output != null) {
|
||||
try {
|
||||
output.close();
|
||||
} catch (IOException e) {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public double getProgress() {
|
||||
return scannedAllPages ? (double) imagesComplete / (double) totalImages
|
||||
: (double) imagesComplete / (double) predictedTotal;
|
||||
}
|
||||
|
||||
public double getScanProgress() {
|
||||
return (double) (pagesScanned) / pagesToScan;
|
||||
}
|
||||
|
||||
public int getImagesDiscovered() {
|
||||
return totalImages;
|
||||
}
|
||||
|
||||
public int getImagesDownloaded() {
|
||||
return imagesComplete;
|
||||
}
|
||||
|
||||
public String getTitle() {
|
||||
return title;
|
||||
}
|
||||
|
||||
public boolean isBusy() {
|
||||
return busy;
|
||||
}
|
||||
|
||||
// you deserve a break
|
||||
// Eugenia Suarez
|
||||
// Mery del Cerro
|
||||
// google them for a good time
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package imgurlibrary;
|
||||
import java.awt.image.BufferedImage;
|
||||
|
||||
|
||||
public interface ImageListener {
|
||||
public abstract void sendImage(BufferedImage image);
|
||||
}
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
package imgurlibrary;
|
||||
|
||||
import java.awt.image.BufferedImage;
|
||||
import java.io.File;
|
||||
|
||||
public abstract class ImgurRequest implements Runnable {
|
||||
public static String LOCAL_ROOT = System.getenv("USERPROFILE") + "\\Desktop\\imgur";
|
||||
protected final String AuthorizationToken = "76535d44f1f94da";
|
||||
public final String name;
|
||||
protected boolean busy;
|
||||
private final ImageListener listener;
|
||||
private volatile int downloaded, discovered, failed;
|
||||
static {
|
||||
if (!new File(LOCAL_ROOT).exists()) {
|
||||
new File(LOCAL_ROOT).mkdirs();
|
||||
}
|
||||
}
|
||||
|
||||
protected synchronized final void discovered(int i) {
|
||||
discovered+=i;
|
||||
}
|
||||
|
||||
protected synchronized final void discovered() {
|
||||
discovered++;
|
||||
}
|
||||
|
||||
protected synchronized final void downloaded(int i) {
|
||||
downloaded+=i;
|
||||
}
|
||||
|
||||
protected synchronized final void downloaded() {
|
||||
downloaded++;
|
||||
}
|
||||
|
||||
protected synchronized final void failed(int i) {
|
||||
failed+=i;
|
||||
}
|
||||
|
||||
protected synchronized final void failed() {
|
||||
failed++;
|
||||
}
|
||||
|
||||
protected void sendImage(BufferedImage image) {
|
||||
if (listener != null)
|
||||
listener.sendImage(image);
|
||||
}
|
||||
|
||||
protected ImgurRequest(String name, ImageListener listener) {
|
||||
busy = true;
|
||||
this.name = name;
|
||||
discovered = 0;
|
||||
downloaded = 0;
|
||||
failed = 0;
|
||||
this.listener = listener;
|
||||
}
|
||||
|
||||
public final double getProgress() {
|
||||
// bc divide by zero error
|
||||
return ((double)(downloaded + failed) / (discovered == 0 ? 1 : discovered));
|
||||
}
|
||||
|
||||
public abstract double getScanProgress(); // idunno how YOU want to
|
||||
// calculate this but uh
|
||||
|
||||
public final boolean isBusy() { // i aint doing it for you so....
|
||||
return busy; // ~All interfaces ever
|
||||
}
|
||||
|
||||
public abstract void run();
|
||||
|
||||
protected final String getExt(String MIME) {
|
||||
switch (MIME) {
|
||||
case "image/jpeg":
|
||||
return ".jpeg";
|
||||
case "image/png":
|
||||
return ".png";
|
||||
case "image/gif":
|
||||
return ".gif";
|
||||
default:
|
||||
return ".jpg";
|
||||
}
|
||||
}
|
||||
|
||||
public int getImagesDiscovered() {
|
||||
return discovered;
|
||||
}
|
||||
|
||||
public int getImagesDownloaded() {
|
||||
return downloaded;
|
||||
}
|
||||
|
||||
public String getTitle() {
|
||||
return name;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,122 @@
|
|||
package imgurlibrary;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.util.Iterator;
|
||||
|
||||
import javax.imageio.ImageIO;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
public class SubredditRequest extends ImgurRequest {
|
||||
private final int pages;
|
||||
private int pagesScanned;
|
||||
private static final String BASE_URL = "https://api.imgur.com/3/gallery/r/";
|
||||
private final String URL;
|
||||
private final String subreddit;
|
||||
private final String LOCAL_ROOT;
|
||||
|
||||
public SubredditRequest(String subreddit, int pages, boolean async) {
|
||||
this(subreddit, pages, async, null);
|
||||
}
|
||||
|
||||
public SubredditRequest(String subreddit, int pages, boolean async, ImageListener listener) {
|
||||
super(subreddit, listener);
|
||||
this.pages = pages;
|
||||
pagesScanned = 0;
|
||||
busy = true;
|
||||
URL = BASE_URL + subreddit + "/time/";
|
||||
this.subreddit = subreddit;
|
||||
LOCAL_ROOT = ImgurRequest.LOCAL_ROOT + File.separatorChar + subreddit;
|
||||
if (!new File(LOCAL_ROOT).exists()) {
|
||||
new File(LOCAL_ROOT).mkdirs();
|
||||
}
|
||||
if (async)
|
||||
new Thread(this).start();
|
||||
else
|
||||
run();
|
||||
}
|
||||
|
||||
@Override
|
||||
public double getScanProgress() {
|
||||
return pagesScanned / (double) (pages);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
// && busy so we can easily exit if a fatal error happens
|
||||
for (int page = 0; page < pages && busy; page++) {
|
||||
|
||||
try {
|
||||
|
||||
String path = "https://api.imgur.com/3/gallery/r/emmawatson/time/0.json";
|
||||
|
||||
HttpURLConnection connection = (HttpURLConnection) ((new URL(path)).openConnection());
|
||||
|
||||
connection.setRequestMethod("GET");
|
||||
connection.addRequestProperty("Authorization", "client-id 76535d44f1f94da");
|
||||
connection.connect();
|
||||
|
||||
if (connection.getResponseCode() == 200) {
|
||||
|
||||
InputStream response = connection.getInputStream();
|
||||
|
||||
savePage(response);
|
||||
|
||||
} else {
|
||||
busy = false;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace(System.out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void savePage(final InputStream response) {
|
||||
new Thread(new Runnable() {
|
||||
public void run() {
|
||||
try {
|
||||
ObjectMapper om = new ObjectMapper();
|
||||
JsonNode root = om.readTree(response);
|
||||
JsonNode data = root.get("data");
|
||||
Iterator<JsonNode> iterator = data.iterator();
|
||||
|
||||
while (iterator.hasNext()) {
|
||||
iterator.next();
|
||||
discovered();
|
||||
}
|
||||
pagesScanned++;
|
||||
|
||||
// reset the iterator ya doof
|
||||
iterator = data.iterator();
|
||||
|
||||
while (iterator.hasNext()) {
|
||||
JsonNode image = iterator.next();
|
||||
// no albums here so no worries
|
||||
String id = image.get("id").asText();
|
||||
String ext = getExt(image.get("type").asText());
|
||||
InputStream in = new URL("http://i.imgur.com/" + id + ext).openConnection().getInputStream();
|
||||
OutputStream out = new FileOutputStream(new File(LOCAL_ROOT + File.separatorChar + id + ext));
|
||||
IOUtils.copy(in, out);
|
||||
downloaded();
|
||||
sendImage(ImageIO.read(new File(LOCAL_ROOT + File.separatorChar + id + ext)));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
failed();
|
||||
}
|
||||
}
|
||||
}).start();
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
new SubredditRequest("derp", 1, true);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue