把对象写入Postgresql中
工作中，遇到把大对象写入Postgresql数据库中
package com.geni_sage.gdme.cws.dic; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.OutputStream; import java.io.Serializable; import java.nio.ByteBuffer; import java.nio.ShortBuffer; import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; import java.sql.Blob; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.HashMap; import com.geni_sage.gdme.cws.kuromoji.dict.CharacterDefinition; import com.geni_sage.gdme.cws.kuromoji.dict.ConnectionCosts; import com.geni_sage.gdme.cws.kuromoji.dict.Dictionaries; import com.geni_sage.gdme.cws.kuromoji.dict.TokenInfoDictionary; import com.geni_sage.gdme.cws.kuromoji.dict.UnknownDictionary; import com.geni_sage.gdme.cws.kuromoji.trie.DoubleArrayTrie; /** * 将日语词典写入维护库 * * @author ywf * */ public class WriteObj2DBTest { static String pathname = "D:\\yuwenfeng\\T_Miner_TextMing\\gdme\\gdm_agent_start\\mode\\segmentdic\\japanesedic"; static String[] dictionaries = { "tid.dat", "tid_map.dat", "unk.dat", "unk_map.dat", "cd.dat", "cc.dat", "dat.dat" }; /** * @param args * @throws ClassNotFoundException * @throws IOException */ public static void main(String[] args) throws IOException, ClassNotFoundException { String driver = "org.postgresql.Driver"; String url = "jdbc:postgresql://127.0.0.1:42856/GDM"; String user = "gsdba"; String password = "gsdba"; if (args.length == 4) { driver = args[0]; url = args[1]; user = args[2]; password = args[3]; } try { Class.forName(driver); } catch (ClassNotFoundException e1) { System.err.println("链接驱动失败"); } Connection con = null; try { con = DriverManager.getConnection(url, user, password); } catch (SQLException e1) { System.err.println("创建连接失败"); } try { createTable(con); write2db(con, dictionaries); // readfromdb(con, "gs_japanesedic", "tid.dat"); // readfromdb(con, "gs_japanesedic", "tid_map.dat"); // readfromdb(con, "gs_japanesedic", "unk.dat"); // readfromdb(con, "gs_japanesedic", "unk_map.dat"); // readfromdb(con, "gs_japanesedic", "cd.dat"); // readfromdb(con, "gs_japanesedic", "cc.dat"); // readfromdb(con, "gs_japanesedic", "dat.dat"); con.close(); } catch (SQLException e1) { e1.printStackTrace(); } catch (Exception e) { e.printStackTrace(); } } /** * 建立日语词典表 * * @param con * @throws SQLException */ private static void createTable(Connection con) throws SQLException { String sql = "create table gs_japanesedic(dictype varchar(100),japaninfo bytea)"; Statement stmt = con.createStatement(); stmt.execute(sql); stmt.close(); } /** * 把流写入维护库 * * @param con * @param path * @param dicname * @throws Exception */ private static void write2db(Connection con, String[] path) throws Exception { CharacterDefinition characterDefinition = UnknownDictionary .getInstance(pathname).getCharacterDefinition(); short[][] cost = ConnectionCosts.getInstance(pathname).getCosts(); String sql = "insert into gs_japanesedic values(?,?)"; PreparedStatement pstmt = con.prepareStatement(sql); for (int i = 0; i < path.length; i++) { pstmt.setString(1, path[i]); byte[] by = null; if (path[i].equals("cd.dat") || path[i].equals("cc.dat")) { if (path[i].startsWith("cd")) {// CharacterDefinition by = getBytes(characterDefinition); } else { by = getBytes(cost);// cost } } else { InputStream input = new FileInputStream(pathname + "/" + path[i]); by = getbytes(input); } pstmt.setBytes(2, by); pstmt.addBatch(); } pstmt.executeBatch(); pstmt.close(); } /** * convet inputstream into byte[] * * @param input * @return * @throws IOException */ private static byte[] getbytes(InputStream input) throws IOException { byte[] by = new byte[input.available()]; input.read(by); return by; } public static DoubleArrayTrie readtrie(InputStream is) throws IOException { DoubleArrayTrie trie = new DoubleArrayTrie(); DataInputStream dis = new DataInputStream(new BufferedInputStream(is)); int baseCheckSize = dis.readInt(); int tailSize = dis.readInt(); ReadableByteChannel channel = Channels.newChannel(dis); ByteBuffer tmpBaseBuffer = ByteBuffer.allocateDirect(baseCheckSize * 4); channel.read(tmpBaseBuffer); tmpBaseBuffer.rewind(); tmpBaseBuffer.asIntBuffer().asReadOnlyBuffer(); ByteBuffer tmpCheckBuffer = ByteBuffer .allocateDirect(baseCheckSize * 4); channel.read(tmpCheckBuffer); tmpCheckBuffer.rewind(); tmpCheckBuffer.asIntBuffer().asReadOnlyBuffer(); ByteBuffer tmpTailBuffer = ByteBuffer.allocateDirect(tailSize * 2); channel.read(tmpTailBuffer); tmpTailBuffer.rewind(); tmpTailBuffer.asCharBuffer().asReadOnlyBuffer(); is.close(); return trie; } private static void readfromdb(Connection con, String table, String dictype) throws SQLException, IOException, ClassNotFoundException { String sql = "select * from " + table + " where dictype = '" + dictype + "'"; PreparedStatement stmt = con.prepareStatement(sql); ResultSet rs = stmt.executeQuery(); while (rs.next()) { String dic = rs.getString(1); byte[] by = rs.getBytes(2); InputStream bi = new ByteArrayInputStream(by); if (dictype.equals("cd.dat") || dictype.equals("cc.dat")) { if (dictype.startsWith("cd")) {// CharacterDefinition CharacterDefinition characterDefinition = (CharacterDefinition) loadObject(bi); } else { short[][] cost = (short[][]) loadObject(bi); } } else if (dictype.startsWith("dat")) { readtrie(bi); } else if (dictype.equals("tid.dat") || dictype.equals("unk.dat")) { loadDictionary(bi); } else { loadTargetMap(bi); } } } protected static void loadDictionary(InputStream is) throws IOException { DataInputStream dis = new DataInputStream(is); int size = dis.readInt(); ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size); ReadableByteChannel channel = Channels.newChannel(is); channel.read(tmpBuffer); is.close(); tmpBuffer.asReadOnlyBuffer(); } protected static void loadTargetMap(InputStream is) throws IOException, ClassNotFoundException { ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream( is)); int[][] targetMap = (int[][]) ois.readObject(); is.close(); } /** * 根据序列化对象得到byte[] * * @param obj * @return * @throws IOException */ public static byte[] getBytes(Serializable obj) throws IOException { ByteArrayOutputStream bout = new ByteArrayOutputStream(); ObjectOutputStream out = new ObjectOutputStream(bout); out.writeObject(obj); out.flush(); byte[] bytes = bout.toByteArray(); bout.close(); out.close(); return bytes; } public static Object getObject(byte[] bytes) throws IOException, ClassNotFoundException { ByteArrayInputStream bi = new ByteArrayInputStream(bytes); ObjectInputStream oi = new ObjectInputStream(bi); Object obj = oi.readObject(); bi.close(); oi.close(); return obj; } public static Object loadObject(InputStream in) throws IOException, ClassNotFoundException { ObjectInputStream oi = new ObjectInputStream(in); Object obj = oi.readObject(); in.close(); oi.close(); return obj; } }
