pse/dpg_backend
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

원천데이터 comtsrcdata_new INSERT 시 batch 적용

Browse Source
This commit is contained in:
psh 2024-06-13 18:24:56 +09:00
parent 7363088d65
commit b2f989e16b
1 changed files with 124 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

234
src/main/java/egovframework/com/mtdata/schema/service/impl/MySQLSchemaViewerImpl.java
View File

@ -6,6 +6,7 @@ import java.util.Map;
import javax.annotation.Resource; import javax.annotation.Resource;
import egovframework.com.mtdata.comtsrcdata.mapper.ComtsrcdataMapper;
import egovframework.com.mtdata.comtsrcdata.service.ComtsrcdataService; import egovframework.com.mtdata.comtsrcdata.service.ComtsrcdataService;
import egovframework.com.mtdata.comtsrcdata.vo.ComtColVO; import egovframework.com.mtdata.comtsrcdata.vo.ComtColVO;
import egovframework.com.mtdata.comtsrcdata.vo.ComtExtrtVO; import egovframework.com.mtdata.comtsrcdata.vo.ComtExtrtVO;
@ -13,9 +14,14 @@ import egovframework.com.mtdata.comtsrcdata.vo.ComtTableVO;
import egovframework.com.mtdata.comtsrcdata.vo.ComtsrcdataVO; import egovframework.com.mtdata.comtsrcdata.vo.ComtsrcdataVO;
import egovframework.com.mtdata.database.vo.ColumnInfoVO; import egovframework.com.mtdata.database.vo.ColumnInfoVO;
import egovframework.com.mtdata.database.vo.TableInfoVO; import egovframework.com.mtdata.database.vo.TableInfoVO;
import org.apache.commons.dbcp2.BasicDataSource;
import org.apache.ibatis.session.ExecutorType;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.json.simple.JSONObject; import org.json.simple.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import egovframework.com.mtdata.database.service.DatabaseService; import egovframework.com.mtdata.database.service.DatabaseService;
@ -28,6 +34,12 @@ public class MySQLSchemaViewerImpl implements DatabaseSchemaViewer {
private static final Logger log = LoggerFactory.getLogger(MySQLSchemaViewerImpl.class); private static final Logger log = LoggerFactory.getLogger(MySQLSchemaViewerImpl.class);
private static final String JDBC_DRIVER = "com.mysql.cj.jdbc.Driver"; private static final String JDBC_DRIVER = "com.mysql.cj.jdbc.Driver";
@Autowired
BasicDataSource dataSource;
@Autowired
private SqlSessionFactory sqlSessionFactory;
@Resource(name = "DatabaseService") @Resource(name = "DatabaseService")
private DatabaseService dbService; private DatabaseService dbService;
@ -50,7 +62,7 @@ public class MySQLSchemaViewerImpl implements DatabaseSchemaViewer {
@Override @Override
public void viewSchema(DatabaseInfoVO dbInfoVO) { public void viewSchema(DatabaseInfoVO dbInfoVO) {
url = "jdbc:mysql://" + dbInfoVO.getDbConnIp() + ":" + dbInfoVO.getDbConnPort() + "/"; url = "jdbc:mysql://" + dbInfoVO.getDbConnIp() + ":" + dbInfoVO.getDbConnPort();
user = dbInfoVO.getDbConnUser(); user = dbInfoVO.getDbConnUser();
password = dbInfoVO.getDbConnPw(); password = dbInfoVO.getDbConnPw();
dbConnId = dbInfoVO.getDbConnId(); dbConnId = dbInfoVO.getDbConnId();
@ -61,38 +73,6 @@ public class MySQLSchemaViewerImpl implements DatabaseSchemaViewer {
log.error("Error while connecting to the database: ", e); log.error("Error while connecting to the database: ", e);
} }
// [테이블 및 컬럼명 가지고 오는 소스]
// try (Connection conn = DriverManager.getConnection(url, user, password)){
// // MySQL database의 이름 추출 - 스키마 조회시 databases catalog에 해당하는 데이터만 출력하기 위함
// Statement stmt = conn.createStatement();
// // MySQL 시스템 데이터베이스는 제외함
// ResultSet databases = stmt.executeQuery("SELECT SCHEMA_NAME AS `Database` FROM information_schema.SCHEMATA WHERE SCHEMA_NAME NOT IN ('information_schema', 'mysql', 'performance_schema', 'sys')");
// String dbNm = null;
// while (databases.next()) {
// dbNm = databases.getString(1);
// System.out.println("dbName : " + dbNm);
// dbInfoVO.setDbNm(dbNm);
// int insertDbList = dbService.insertDbInfo(dbInfoVO);
//
// DatabaseMetaData metaData = conn.getMetaData();
// ResultSet tables = metaData.getTables(dbNm, null, "%", null); //catalog, schemaPattern, tableNamePattern, types
// // 여기서 부터 필요한 테이블의 정보를 출력함
// while(tables.next()) {
// String tableName = tables.getString(3);
// System.out.println("Table : " + tableName);
//
// ResultSet columns = metaData.getColumns(dbNm, null, tableName, "%");
// while (columns.next()) {
// String columnName = columns.getString(4);
// String columnType = columns.getString(6);
// System.out.println("\tColumn: " + columnName+ " (" + columnType + ")");
// }
//
// }
// }
// } catch (Exception e) {
// e.printStackTrace();
// }
} }
// 기관정보 스키마 // 기관정보 스키마
@ -191,10 +171,13 @@ public class MySQLSchemaViewerImpl implements DatabaseSchemaViewer {
} }
// 원천데이터 처리 // 원천데이터 처리
private void comtsrcProcess(Connection conn, DatabaseInfoVO dbVo, TableInfoVO tbVo) throws SQLException { private void comtsrcProcess(Connection conn, DatabaseInfoVO dbVo, TableInfoVO tbVo) throws SQLException {
try { try {
SqlSession sqlSession = null;
sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH);
// [ comtextrtblinfo ] 원천데이터 추출 정보 저장 // [ comtextrtblinfo ] 원천데이터 추출 정보 저장
ComtExtrtVO comExVo = new ComtExtrtVO(); ComtExtrtVO comExVo = new ComtExtrtVO();
comExVo.setSysId(dbVo.getSysId()); comExVo.setSysId(dbVo.getSysId());
@ -217,7 +200,7 @@ public class MySQLSchemaViewerImpl implements DatabaseSchemaViewer {
ResultSetMetaData rsmd = datas.getMetaData(); ResultSetMetaData rsmd = datas.getMetaData();
int colCnt = rsmd.getColumnCount(); int colCnt = rsmd.getColumnCount();
JSONObject obj = new JSONObject(); JSONObject obj = new JSONObject();
int insertDataCnt = 0; // int insertDataCnt = 0;
int insertColCnt = 0; int insertColCnt = 0;
// [ comtsrcdatacol_new ] 원천테이블의 컬럼 정보 저장 // [ comtsrcdatacol_new ] 원천테이블의 컬럼 정보 저장
@ -241,14 +224,20 @@ public class MySQLSchemaViewerImpl implements DatabaseSchemaViewer {
ComtsrcdataVO srcVo = new ComtsrcdataVO(); ComtsrcdataVO srcVo = new ComtsrcdataVO();
srcVo.setSrcDataVl(obj.toString()); srcVo.setSrcDataVl(obj.toString());
srcVo.setSrcDataMetaId(comtTableVo.getSrcDataMetaId()); srcVo.setSrcDataMetaId(comtTableVo.getSrcDataMetaId());
insertDataCnt = comtsrcdataService.insertComtSrcdata(srcVo); // mybatis batch 처리
sqlSession.getMapper(ComtsrcdataMapper.class).insertComtSrcdata(srcVo);
if(insertDataCnt > 0){ // batch 처리 전
log.info("all data insert SUCCESS!!!!!!"); // insertDataCnt = comtsrcdataService.insertComtSrcdata(srcVo);
}else{ // if(insertDataCnt > 0){
log.warn("No DATA was inserted for: {}", tbNm); // log.info("all data insert SUCCESS!!!!!!");
} // }else{
// log.warn("No DATA was inserted for: {}", tbNm);
// }
} }
sqlSession.flushStatements();
sqlSession.commit();
}else{ }else{
log.warn("No COLUMN info was inserted for: {}", tbNm); log.warn("No COLUMN info was inserted for: {}", tbNm);
} }
@ -264,76 +253,101 @@ public class MySQLSchemaViewerImpl implements DatabaseSchemaViewer {
} }
// 원천데이터 -> 공유데이터 INSERT // 원천데이터 배치처리 (JAVA)
private void comtrlsProcess(Connection conn, DatabaseInfoVO dbVo, TableInfoVO tbVo) throws SQLException { // private void comtsrcBatchProcess_org(Connection conn, DatabaseInfoVO dbVo, TableInfoVO tbVo) throws SQLException {
try { // PreparedStatement pstmt = null;
// Connection insertCon = null;
//
// try {
// Class.forName("org.postgresql.Driver");
// insertCon=dataSource.getConnection();
//
// // [ comtextrtblinfo ] 원천데이터 추출 정보 저장
// ComtExtrtVO comExVo = new ComtExtrtVO();
// comExVo.setSysId(dbVo.getSysId());
// int extCnt = comtsrcdataService.insertComtExtrtInfo(comExVo);
//
// if (extCnt > 0) {
// String dbNm = dbVo.getDbNm();
// String tbNm = tbVo.getTableNm();
//
// // [ comtsrcdatameta_new ] 원천테이블의 테이블 정보 저장
// ComtTableVO comtTableVo = new ComtTableVO();
// comtTableVo.setExtrId(comExVo.getExtrId());
// comtTableVo.setSrcDataTblNm(tbNm);
// int insertTbCnt = comtsrcdataService.insertComtTabledataInfo(comtTableVo);
//
// if (insertTbCnt > 0) {
// String selectTableSql = "SELECT * FROM "+dbNm+"."+tbNm;
// Statement stmt = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_UPDATABLE);
// ResultSet datas = stmt.executeQuery(selectTableSql);
// ResultSetMetaData rsmd = datas.getMetaData();
// int colCnt = rsmd.getColumnCount();
// JSONObject obj = new JSONObject();
// int insertDataCnt = 0;
// int insertColCnt = 0;
//
// // [ comtsrcdatacol_new ] 원천테이블의 컬럼 정보 저장
// for (int i=0; i<colCnt; i++) {
// ComtColVO colVo = new ComtColVO();
// colVo.setSrcDataColNm(rsmd.getColumnLabel(i+1));
// colVo.setSrcDataMetaId(comtTableVo.getSrcDataMetaId());
// insertColCnt = comtsrcdataService.insertComtSrcCol(colVo);
// }
//
// // [ comtsrcdata_new ] 원천데이터 저장
// if(insertColCnt > 0){
// String sql = "INSERT INTO dpc.public.comtsrcdata_new (src_data_id, src_data_meta_id, src_data_vl) values(nextval(\'comtsrcdata_new_src_data_id_seq\'), ?, ?::jsonb)";
// pstmt = insertCon.prepareStatement(sql);
//
// while (datas.next()){
// // 컬럼이름을 key , 데이터를 value에 넣어서 json 형태로 생성
// Map<String, String> dataVlMap = new HashMap<>();
// for (int i=0; i<colCnt; i++) {
// String columnName = rsmd.getColumnLabel(i+1);
// String value = datas.getString(i+1);
// dataVlMap.put(columnName, value);
// }
// obj.putAll(dataVlMap);
//
// ComtsrcdataVO srcVo = new ComtsrcdataVO();
// srcVo.setSrcDataMetaId(comtTableVo.getSrcDataMetaId());
// srcVo.setSrcDataVl(obj.toString());
//
// // batch setting
// pstmt.setInt(1, srcVo.getSrcDataMetaId());
// pstmt.setString(2, srcVo.getSrcDataVl());
// pstmt.addBatch();
// pstmt.clearParameters();
//
// if(datas.last()){
// pstmt.executeBatch();
// pstmt.clearBatch();
// insertCon.commit();
// log.info("pstmt:", pstmt);
// log.info("all data insert SUCCESS!!!!!!");
// }
// }
// pstmt.executeBatch();
// insertCon.commit();
//
// }else{
// log.warn("No COLUMN info was inserted for: {}", tbNm);
// }
// }else{
// log.warn("No Table info was inserted for: {}", tbNm);
// }
// }
//
// } catch (Exception e) {
// e.printStackTrace();
// insertCon.rollback();
// } finally {
// if(pstmt != null) pstmt.close();
// if(insertCon != null) insertCon.close();
// }
// }
ComtExtrtVO comExVo = new ComtExtrtVO();
comExVo.setSysId(dbVo.getSysId());
int extCnt = comtsrcdataService.insertComtExtrtInfo(comExVo);
if (extCnt > 0) {
String dbNm = dbVo.getDbNm();
String tbNm = tbVo.getTableNm();
// [ comtrlstrgtmetadata ] 공유데이터 메타정보 테이블
ComtTableVO comtTableVo = new ComtTableVO();
comtTableVo.setExtrId(comExVo.getExtrId());
comtTableVo.setSrcDataTblNm(tbNm);
int insertTbCnt = comtsrcdataService.insertComtTabledataInfo(comtTableVo);
if (insertTbCnt > 0) {
String selectTableSql = "SELECT * FROM "+dbNm+"."+tbNm;
Statement stmt = conn.createStatement();
ResultSet datas = stmt.executeQuery(selectTableSql);
ResultSetMetaData rsmd = datas.getMetaData();
int colCnt = rsmd.getColumnCount();
JSONObject obj = new JSONObject();
int insertDataCnt = 0;
int insertColCnt = 0;
// [ comtrlsdatacol ] 공유데이터 컬럼 정보 테이블
for (int i=0; i<colCnt; i++) {
ComtColVO colVo = new ComtColVO();
colVo.setSrcDataColNm(rsmd.getColumnLabel(i+1));
colVo.setSrcDataMetaId(comtTableVo.getSrcDataMetaId());
insertColCnt = comtsrcdataService.insertComtSrcCol(colVo);
}
// [ comtrlsdata ] 공유데이터 테이블
if(insertColCnt > 0){
while (datas.next()){
Map<String, String> dataVlMap = new HashMap<>();
for (int i=0; i<colCnt; i++) {
String columnName = rsmd.getColumnLabel(i+1);
String value = datas.getString(i+1);
dataVlMap.put(columnName, value);
}
obj.putAll(dataVlMap);
ComtsrcdataVO srcVo = new ComtsrcdataVO();
srcVo.setSrcDataVl(obj.toString());
srcVo.setSrcDataMetaId(comtTableVo.getSrcDataMetaId());
insertDataCnt = comtsrcdataService.insertComtSrcdata(srcVo);
if(insertDataCnt > 0){
log.info("all data insert SUCCESS!!!!!!");
}else{
log.warn("No DATA was inserted for: {}", tbNm);
}
}
}else{
log.warn("No COLUMN info was inserted for: {}", tbNm);
}
}else{
log.warn("No Table info was inserted for: {}", tbNm);
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
} }