이클립스(eclipse)에서 웹 크롤링 with MongoDB

몽고 DB와 연결하기 위해 ODBC, JDBC같은 library가 필요하다. 다운 받아야 하는 것은 mongo-java-driver

https://oss.sonatype.org/content/repositories/releases/org/mongodb/mongo-java-driver/3.7.0/ 에 접속해서

mongo-java-driver-3.7.0.jar 다운

이클립스 상의 java project에서 properties 들어가고 java build path -> library탭 클릭 -> external jars 선택하고 다운 받은 .jar 파일 선택하고 적용

몽고 DB 연동해서 지난 번에 한 웹 크롤링 결과를 저장한다.

package crawler;

import java.io.IOException;

import java.util.ArrayList;

import java.util.List;

import java.util.logging.Level;

import java.util.logging.Logger;

import org.bson.Document;

import org.jsoup.Jsoup;

import org.jsoup.select.Elements;

import com.mongodb.MongoClient;

import com.mongodb.MongoClientURI;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.MongoDatabase;

import com.mongodb.client.model.IndexOptions;

public class mongocon {

public static void lgnLog() {

Logger mongoLogger = Logger.getLogger("org.mongodb.driver");

mongoLogger.setLevel(Level.SEVERE);

}

public static void main(String[] args)throws IOException {

lgnLog();

MongoClient mongoClient = new MongoClient(new MongoClientURI("mongodb://localhost:27017"));

MongoDatabase database = mongoClient.getDatabase("Sample"); //use 또는 create함

MongoCollection<Document> collection = database.getCollection("samples");

String url="https://www.zillow.com/homes/for_sale/Denver-CO/pmf,pf_pt/118576962_zpid/11093_rid/globalrelevanceex_sort/39.976331,-104.6101,39.554089,-105.104485_rect/10_zm/";

org.jsoup.nodes.Document document = Jsoup.connect(url).get();//url 해석하기

Elements price = document.getElementsByClass("zsg-photo-card-price");

Elements addr = document.getElementsByClass("zsg-photo-card-address");

List<Document> documents = new ArrayList<Document>();

for(int i = 1; i <= price.size(); i++) { //부모 클래스 안에 있는 address, price 클래스 출력

Document indx = new Document("i",i); //인덱스를 만들기

collection.createIndex(indx, new IndexOptions().unique(true)); //유일한 인덱스, primary key

documents.add(indx.append("price",price.get(i-1).text()).append("address",addr.get(i-1).text()));

}

collection.insertMany(documents);

//collection.drop();

}

}

다음은 몽고DB collection에 있는 값 찾고 수정하는 코드.

package crawler;

import static com.mongodb.client.model.Filters.eq;

import static com.mongodb.client.model.Updates.inc;

import java.util.ArrayList;

import java.util.List;

import java.util.logging.Level;

import java.util.logging.Logger;

import org.bson.Document;

import com.mongodb.Block;

import com.mongodb.MongoClient;

import com.mongodb.client.MongoCollection;

import com.mongodb.client.MongoCursor;

import com.mongodb.client.MongoDatabase;

import com.mongodb.client.result.UpdateResult;

public class collectionFind {

public static void lgnLog() {

Logger mongoLogger = Logger.getLogger("org.mongodb.driver");

mongoLogger.setLevel(Level.SEVERE);

}

public static void main(String[] args) {

lgnLog();

MongoClient mongoClient = new MongoClient("localhost",27017);

MongoDatabase database = mongoClient.getDatabase("Sample");

MongoCollection<Document> collection = database.getCollection("find");

List<Document> documents = new ArrayList<Document>();

/*

for(int i=1;i<=3;i++) {//인덱스는 0이 될 수 없다고 한다.

Document indx = new Document("i",i);

collection.createIndex(indx, new IndexOptions().unique(true));

documents.add(indx

.append("value:",i));

}

collection.insertMany(documents);

*/

//MongoDB에 있는 find collection에 있는 document를 가져온다.

MongoCursor<Document> cursor = collection.find().iterator();

try {

while(cursor.hasNext()) {

System.out.println(cursor.next().toJson());

}

}finally {

cursor.close();

}

for(Document cur:collection.find()) { //collection 끝까지 돌면서 찾기

System.out.println(cur.toJson());

}

Document myDoc = collection.find(eq("i", 1)).first(); //인덱스가 1인 것 중 첫번째

    System.out.println(myDoc.toJson());

    

    Block<Document> printBlock = new Block<Document>() {

    @Override

    public void apply(final Document document) {

    System.out.println("block:"+document.toJson());

    }

    };

    

    collection.updateOne(eq("i", 2), //인덱스가 2인 것을 6으로 변경

    new Document("$set", new Document("i",6)));

    

    UpdateResult updateResult =

    collection.updateMany(eq("i",2), //인덱스 2인 것에 4를 더함.

    inc("i",4));

}

}