WikiTableParser.java

package com.osroyale.util.tools.wiki.parser;
 
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import org.apache.logging.log4j.LogManager;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
 
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.LinkedList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

 
public abstract class WikiTableParser {

    private static final Path WRITE_PATH = Paths.get("./data/wiki");

    protected static final String WIKI_LINK = "http://oldschoolrunescape.wikia.com/wiki/";
 
    private static final org.apache.logging.log4j.Logger logger = LogManager.getLogger(WikiTableParser.class);

    protected static final Gson GSON = new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create();

    private final ListeningExecutorService executorService;

    protected final LinkedList<WikiTable> tables;

    public WikiTableParser(LinkedList<WikiTable> tables) {
        this.tables = tables;
        ExecutorService delegateService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors(),
                new ThreadFactoryBuilder().setNameFormat("WikiTableParserThread").build());
        executorService = MoreExecutors.listeningDecorator(delegateService);
    }

    public void begin() throws InterruptedException {
        int size = tables.size();
        logger.info("Parsing " + size + " names from " + WIKI_LINK);
 
        tables.forEach(table -> execute(new ParserTask(table)));
 
        executorService.shutdown();
        executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
 
        finish();
    }
 
    protected void execute(Runnable runnable) {
        executorService.execute(runnable);
    }
 
    protected abstract void finish();

    protected static void writeToJson(String name, JsonElement array) {
        try {
            String json = GSON.toJson(array);
 
            if (!Files.exists(WRITE_PATH)) {
                Files.createDirectory(WRITE_PATH);
            }
 
            Path path = WRITE_PATH.resolve(name += ".json");
            Files.write(path, json.getBytes());
 
            logger.info("Successfully wrote " + path);
        } catch (IOException e) {
            logger.warn("Could not save '" + name + "' to '" + WRITE_PATH + "'", e);
        }
    }

    private final class ParserTask implements Runnable {

        private final WikiTable table;

        private ParserTask(WikiTable table) {
            this.table = table;
        }
 
        @Override
        public void run() {
            try {
                String link = table.getLink();
                Document document = Jsoup.connect(link).get();
                table.parseDocument(document);
            } catch (Exception e) {
                System.out.println("Could not parse table from wiki for '" + table.getLink() + "'");
//                logger.error("Could not parse table from wiki for '" + table.getLink() + "'", e);
            }
        }
    }
 
}