1598813d80595106462b28ea355eb277822fa74f
[sqoop.git] / src / test / org / apache / sqoop / manager / oracle / OraOopTestCase.java
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 package org.apache.sqoop.manager.oracle;
20
21 import java.io.StringWriter;
22 import java.sql.Connection;
23 import java.sql.DriverManager;
24 import java.sql.PreparedStatement;
25 import java.sql.ResultSet;
26 import java.sql.SQLException;
27 import java.sql.Statement;
28 import java.util.ArrayList;
29 import java.util.HashMap;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.Map.Entry;
33 import java.util.regex.Matcher;
34 import java.util.regex.Pattern;
35
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.log4j.Layout;
39 import org.apache.log4j.Logger;
40 import org.apache.log4j.PatternLayout;
41 import org.apache.log4j.WriterAppender;
42 import org.apache.sqoop.manager.oracle.util.HadoopFiles;
43 import org.apache.sqoop.manager.oracle.util.OracleData;
44
45 import org.apache.sqoop.Sqoop;
46 import org.apache.sqoop.manager.oracle.util.OracleUtils;
47 import org.apache.sqoop.testcategories.sqooptest.IntegrationTest;
48 import org.apache.sqoop.testcategories.thirdpartytest.OracleTest;
49 import org.junit.experimental.categories.Category;
50
51 import static org.junit.Assert.assertEquals;
52
53 /**
54 * Base test case for OraOop to handle common functions.
55 */
56 @Category({IntegrationTest.class, OracleTest.class})
57 public abstract class OraOopTestCase {
58
59 private static final OraOopLog LOG = OraOopLogFactory.getLog(
60 OraOopTestCase.class.getName());
61 private static final boolean ESCAPING_DISABLED_DEFAULT = true;
62 private String sqoopGenLibDirectory = System.getProperty("user.dir")
63 + "/target/tmp/lib";
64 private String sqoopGenSrcDirectory = System.getProperty("user.dir")
65 + "/target/tmp/src";
66 private String sqoopTargetDirectory = "target/tmp/";
67 private String sqoopGenClassName = "org.apache.sqoop.gen.OraOopTestClass";
68
69 private Connection conn;
70
71 protected ClassLoader classLoader;
72 {
73 classLoader = Thread.currentThread().getContextClassLoader();
74 if (classLoader == null) {
75 classLoader = OraOopTestCase.class.getClassLoader();
76 }
77 }
78
79 static {
80 Configuration
81 .addDefaultResource(OraOopConstants.ORAOOP_SITE_TEMPLATE_FILENAME);
82 Configuration.addDefaultResource(OraOopConstants.ORAOOP_SITE_FILENAME);
83 }
84
85 protected String getSqoopTargetDirectory() {
86 return sqoopTargetDirectory;
87 }
88
89 protected void setSqoopTargetDirectory(String newSqoopTargetDirectory) {
90 this.sqoopTargetDirectory = newSqoopTargetDirectory;
91 }
92
93 protected String getSqoopGenLibDirectory() {
94 return sqoopGenLibDirectory;
95 }
96
97 protected String getSqoopGenSrcDirectory() {
98 return sqoopGenSrcDirectory;
99 }
100
101 protected String getSqoopGenClassName() {
102 return sqoopGenClassName;
103 }
104
105 protected void setSqoopGenClassName(String sqoopGenClassName) {
106 this.sqoopGenClassName = sqoopGenClassName;
107 }
108
109 protected Connection getTestEnvConnection() throws SQLException {
110 if (this.conn == null) {
111 this.conn =
112 DriverManager.getConnection(OracleUtils.CONNECT_STRING,
113 OracleUtils.ORACLE_USER_NAME, OracleUtils.ORACLE_USER_PASS);
114 this.conn.setAutoCommit(false);
115 }
116 return this.conn;
117 }
118
119 protected void closeTestEnvConnection() {
120 try {
121 if (this.conn != null) {
122 this.conn.close();
123 }
124 } catch (SQLException e) {
125 // Tried to close connection but failed - continue anyway
126 }
127 this.conn = null;
128 }
129
130 protected void createTable(String fileName, boolean dropTableIfExists) {
131 try {
132 Connection localConn = getTestEnvConnection();
133 int parallelProcesses = OracleData.getParallelProcesses(localConn);
134 int rowsPerSlave =
135 OracleUtils.INTEGRATIONTEST_NUM_ROWS / parallelProcesses;
136 try {
137 long startTime = System.currentTimeMillis();
138 OracleData.createTable(localConn, fileName, parallelProcesses,
139 rowsPerSlave, dropTableIfExists);
140 LOG.debug("Created and loaded table in "
141 + ((System.currentTimeMillis() - startTime) / 1000) + " seconds.");
142 } catch (SQLException e) {
143 if (e.getErrorCode() == 955) {
144 LOG.debug("Table already exists - using existing data");
145 } else {
146 throw new RuntimeException(e);
147 }
148 }
149 } catch (Exception e) {
150 throw new RuntimeException(e);
151 }
152 }
153
154 protected void createTable(String fileName) {
155 createTable(fileName, false);
156 }
157
158 protected void createTableFromSQL(String sql, String tableName) throws SQLException {
159 Connection conn = getTestEnvConnection();
160 try (Statement stmt = conn.createStatement()) {
161 stmt.execute("BEGIN EXECUTE IMMEDIATE 'DROP TABLE " + tableName
162 + "'; EXCEPTION WHEN OTHERS THEN IF SQLCODE != -942 THEN RAISE; END IF; END;");
163 stmt.execute(sql);
164 }
165 }
166
167 protected int countTable(String inputTableName, List<String> partitionList) {
168 String tableName = org.apache.sqoop.manager.oracle.OracleUtils.escapeIdentifier(inputTableName);
169 if (tableName.startsWith("\"\"") && tableName.endsWith("\"\"")) {
170 // Table names may be double quoted to work around Sqoop issue
171 tableName = tableName.substring(1, tableName.length() - 1);
172 }
173 String sql = null;
174 int numRows = 0;
175 if (partitionList != null && partitionList.size() > 0) {
176 sql = "SELECT sum(cnt) FROM (";
177 int i = 0;
178 for (String partition : partitionList) {
179 i++;
180 if (i > 1) {
181 sql += " UNION ALL ";
182 }
183 sql +=
184 "SELECT count(*) cnt FROM " + tableName + " PARTITION(\""
185 + partition + "\")";
186 }
187 sql += ")";
188 } else {
189 sql = "SELECT count(*) FROM " + tableName;
190 }
191 try {
192 PreparedStatement stmt =
193 this.getTestEnvConnection().prepareStatement(sql);
194 stmt.execute();
195 ResultSet results = stmt.getResultSet();
196 results.next();
197 numRows = results.getInt(1);
198 } catch (SQLException e) {
199 throw new RuntimeException("Could not count number of rows in table "
200 + tableName, e);
201 }
202 return numRows;
203 }
204
205 protected Configuration getSqoopConf() {
206 Configuration sqoopConf = new Configuration();
207 return sqoopConf;
208 }
209
210 protected int runImport(String tableName, Configuration sqoopConf,
211 boolean sequenceFile) {
212 return runImport(tableName, sqoopConf, sequenceFile, ESCAPING_DISABLED_DEFAULT);
213 }
214
215 protected int runImport(String tableName, Configuration sqoopConf,
216 boolean sequenceFile, boolean escapingDisabled) {
217 Logger rootLogger = Logger.getRootLogger();
218 StringWriter stringWriter = new StringWriter();
219 Layout layout = new PatternLayout("%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n");
220 WriterAppender writerAppender = new WriterAppender(layout, stringWriter);
221 rootLogger.addAppender(writerAppender);
222
223 List<String> sqoopArgs = new ArrayList<String>();
224
225 sqoopArgs.add("import");
226
227 sqoopArgs.add("--direct");
228
229 if (sequenceFile) {
230 sqoopArgs.add("--as-sequencefile");
231 }
232
233 sqoopArgs.add("--connect");
234 sqoopArgs.add(OracleUtils.CONNECT_STRING);
235
236 sqoopArgs.add("--username");
237 sqoopArgs.add(OracleUtils.ORACLE_USER_NAME);
238
239 sqoopArgs.add("--password");
240 sqoopArgs.add(OracleUtils.ORACLE_USER_PASS);
241
242 sqoopArgs.add("--table");
243 sqoopArgs.add(tableName);
244
245 sqoopArgs.add("--target-dir");
246 sqoopArgs.add(this.sqoopTargetDirectory);
247
248 sqoopArgs.add("--class-name");
249 sqoopArgs.add(getSqoopGenClassName());
250
251 sqoopArgs.add("--bindir");
252 sqoopArgs.add(this.sqoopGenLibDirectory);
253
254 sqoopArgs.add("--outdir");
255 sqoopArgs.add(this.sqoopGenSrcDirectory);
256
257 sqoopArgs.add("--oracle-escaping-disabled");
258 sqoopArgs.add(Boolean.toString(escapingDisabled));
259
260 if (OracleUtils.NUM_MAPPERS != 0) {
261 sqoopArgs.add("--num-mappers");
262 sqoopArgs.add(Integer.toString(OracleUtils.NUM_MAPPERS));
263 }
264
265 int rowsInTable =
266 countTable(tableName, OraOopUtilities.splitOracleStringList(sqoopConf
267 .get(OraOopConstants.ORAOOP_IMPORT_PARTITION_LIST)));
268
269 int retCode =
270 Sqoop.runTool(sqoopArgs.toArray(new String[sqoopArgs.size()]),
271 sqoopConf);
272 int rowsImported = 0;
273 if (retCode == 0) {
274 String logString = stringWriter.toString();
275 Pattern pattern =
276 Pattern.compile(
277 "(INFO mapreduce.ImportJobBase: Retrieved )([0-9]+)( records.)");
278 Matcher matcher = pattern.matcher(logString);
279 while (matcher.find()) {
280 rowsImported = Integer.parseInt(matcher.group(2));
281 }
282 }
283 assertEquals("Incorrect number of rows imported", rowsInTable,
284 rowsImported);
285 return retCode;
286 }
287
288 protected int runExportFromTemplateTable(String templateTableName,
289 String tableName, boolean isPartitoned) {
290 return runExportFromTemplateTable(templateTableName, tableName, isPartitoned, ESCAPING_DISABLED_DEFAULT);
291 }
292
293 protected int runExportFromTemplateTable(String templateTableName,
294 String tableName, boolean isPartitoned, boolean escapingDisabled) {
295 Map<String, String> stringConfigEntries = new HashMap<String, String>();
296 stringConfigEntries.put("oraoop.template.table", templateTableName);
297 Map<String, Boolean> booleanConfigEntries = new HashMap<String, Boolean>();
298 booleanConfigEntries.put("oraoop.partitioned", isPartitoned);
299
300 return runExport(tableName, new ArrayList<String>(), stringConfigEntries, booleanConfigEntries, escapingDisabled);
301 }
302
303 protected int runExport(String tableName, List<String> additionalArgs, Map<String, String> stringConfigEntries,
304 Map<String, Boolean> booleanConfigEntries, boolean escapingDisabled) {
305 List<String> sqoopArgs = new ArrayList<String>();
306
307 sqoopArgs.add("export");
308
309 sqoopArgs.add("--direct");
310
311 sqoopArgs.add("--connect");
312 sqoopArgs.add(OracleUtils.CONNECT_STRING);
313
314 sqoopArgs.add("--username");
315 sqoopArgs.add(OracleUtils.ORACLE_USER_NAME);
316
317 sqoopArgs.add("--password");
318 sqoopArgs.add(OracleUtils.ORACLE_USER_PASS);
319
320 sqoopArgs.add("--table");
321 sqoopArgs.add(tableName);
322
323 sqoopArgs.add("--export-dir");
324 sqoopArgs.add(this.sqoopTargetDirectory);
325
326 sqoopArgs.add("--class-name");
327 sqoopArgs.add(getSqoopGenClassName());
328
329 sqoopArgs.add("--bindir");
330 sqoopArgs.add(this.sqoopGenLibDirectory);
331
332 sqoopArgs.add("--outdir");
333 sqoopArgs.add(this.sqoopGenSrcDirectory);
334
335 sqoopArgs.add("--oracle-escaping-disabled");
336 sqoopArgs.add(Boolean.toString(escapingDisabled));
337
338 sqoopArgs.addAll(additionalArgs);
339
340 Configuration sqoopConf = getSqoopConf();
341
342 sqoopConf.setBoolean("oraoop.drop.table", true);
343 sqoopConf.setBoolean("oraoop.nologging", true);
344
345 for (Entry<String, String> entry : stringConfigEntries.entrySet()) {
346 sqoopConf.set(entry.getKey(), entry.getValue());
347 }
348
349 for (Entry<String, Boolean> entry : booleanConfigEntries.entrySet()) {
350 sqoopConf.setBoolean(entry.getKey(), entry.getValue());
351 }
352
353 return Sqoop.runTool(sqoopArgs.toArray(new String[sqoopArgs.size()]), sqoopConf);
354 }
355
356 protected int runCompareTables(Connection connection, String table1,
357 String table2) throws SQLException {
358 PreparedStatement stmt;
359 stmt = connection.prepareStatement(
360 "select count(*) from (select * from (select * from "
361 + table1
362 + " minus select * from "
363 + table2
364 + ") union all select * from (select * from "
365 + table2
366 + " minus select * from " + table1 + "))");
367 ResultSet results = stmt.executeQuery();
368 results.next();
369 int numDifferences = results.getInt(1);
370 return numDifferences;
371 }
372
373 protected void cleanupFolders() throws Exception {
374 HadoopFiles.delete(new Path(getSqoopTargetDirectory()), true);
375 HadoopFiles.delete(new Path(getSqoopGenSrcDirectory()), true);
376 HadoopFiles.delete(new Path(getSqoopGenLibDirectory()), true);
377 }
378
379 }