ed949b98f9d96fcf188786823d0fed7dd5cf3e43
[sqoop.git] / src / test / org / apache / sqoop / manager / db2 / DB2ImportAllTableWithSchemaManualTest.java
1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18 package org.apache.sqoop.manager.db2;
19
20 import java.io.BufferedReader;
21 import java.io.File;
22 import java.io.FileInputStream;
23 import java.io.IOException;
24 import java.io.InputStreamReader;
25 import java.sql.Connection;
26 import java.sql.SQLException;
27 import java.sql.Statement;
28 import java.util.ArrayList;
29
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.io.IOUtils;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.sqoop.manager.Db2Manager;
37 import org.apache.sqoop.testcategories.thirdpartytest.Db2Test;
38 import org.apache.sqoop.testcategories.sqooptest.ManualTest;
39 import org.apache.sqoop.tool.ImportAllTablesTool;
40 import org.apache.sqoop.Sqoop;
41 import org.junit.After;
42 import org.junit.Before;
43 import org.junit.Test;
44
45
46 import org.apache.sqoop.SqoopOptions;
47 import org.apache.sqoop.testutil.CommonArgs;
48 import org.apache.sqoop.testutil.ImportJobTestCase;
49 import org.apache.sqoop.tool.SqoopTool;
50 import org.apache.sqoop.util.FileListing;
51 import org.apache.sqoop.util.LoggingUtils;
52 import org.junit.experimental.categories.Category;
53
54 import static org.junit.Assert.assertEquals;
55 import static org.junit.Assert.assertTrue;
56 import static org.junit.Assert.fail;
57
58 /**
59 * Test the DB2 XML data type.
60 *
61 * This uses JDBC to import data from an DB2 database into HDFS.
62 *
63 * Since this requires an DB2 Server installation,
64 * this class is named in such a way that Sqoop's default QA process does
65 * not run it. You need to run this manually with
66 * -Dtestcase=DB2ImportAllTableWithSchema
67
68 * You need to put DB2 JDBC driver library (db2jcc.jar) in a location
69 * where Sqoop will be able to access it (since this library cannot be checked
70 * into Apache's tree for licensing reasons).
71 *
72 * To set up your test environment:
73 * Install DB2 Express 9.7 C server.
74 * Create a database SQOOP
75 * Create a login SQOOP with password PASSWORD and grant all
76 * access for database SQOOP to user SQOOP.
77 */
78 @Category({ManualTest.class, Db2Test.class})
79 public class DB2ImportAllTableWithSchemaManualTest extends ImportJobTestCase {
80
81 public static final Log LOG = LogFactory.getLog(
82 DB2ImportAllTableWithSchemaManualTest.class.getName());
83
84 static final String HOST_URL = System.getProperty(
85 "sqoop.test.db2.connectstring.host_url",
86 "jdbc:db2://9.30.245.234:60000");
87
88 static final String DATABASE_NAME = System.getProperty(
89 "sqoop.test.db2.connectstring.database",
90 "TESTDB");
91 static final String DATABASE_USER = System.getProperty(
92 "sqoop.test.db2.connectstring.username",
93 "DB2FENC1");
94 static final String DATABASE_PASSWORD = System.getProperty(
95 "sqoop.test.db2.connectstring.password",
96 "DB2FENC1");
97
98 static final String TABLE_NAME = "TEST.COMPANY";
99 static final String TABLE_SCHEMA = "TEST";
100 static final String CONNECT_STRING = HOST_URL
101 + "/" + DATABASE_NAME;
102 static String ExpectedResults =
103 "1,doc1";
104
105
106 String [] extraArgs = { "--",
107 "--schema", TABLE_SCHEMA,
108 };
109
110 static {
111 LOG.info("Using DB2 CONNECT_STRING HOST_URL is : "+HOST_URL);
112 LOG.info("Using DB2 CONNECT_STRING: " + CONNECT_STRING);
113 }
114
115 // instance variables populated during setUp, used during tests
116 private Db2Manager manager;
117
118 protected String getTableName() {
119 return TABLE_NAME;
120 }
121
122
123 @Before
124 public void setUp() {
125 super.setUp();
126
127 SqoopOptions options = new SqoopOptions(CONNECT_STRING, getTableName());
128 options.setUsername(DATABASE_USER);
129 options.setPassword(DATABASE_PASSWORD);
130
131 manager = new Db2Manager(options);
132
133 // Drop the existing table, if there is one.
134 Connection conn = null;
135 Statement stmt = null;
136 try {
137 conn = manager.getConnection();
138 stmt = conn.createStatement();
139 stmt.execute("DROP TABLE " + getTableName());
140 } catch (SQLException sqlE) {
141 LoggingUtils.logAll(LOG, "Table was not dropped: ", sqlE);
142 } finally {
143 try {
144 if (null != stmt) {
145 stmt.close();
146 }
147 } catch (Exception ex) {
148 LOG.warn("Exception while closing stmt", ex);
149 }
150 }
151
152 // Create and populate table
153 try {
154 conn = manager.getConnection();
155 conn.setAutoCommit(false);
156 stmt = conn.createStatement();
157
158 // create the database table and populate it with data.
159 stmt.executeUpdate("CREATE TABLE " + getTableName() + " ("
160 + "ID int, "
161 + "DOCNAME VARCHAR(20))");
162
163 stmt.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
164 + "1,'doc1' "
165 + " )");
166 conn.commit();
167 } catch (SQLException sqlE) {
168 LoggingUtils.logAll(LOG, "Encountered SQL Exception: ", sqlE);
169 } finally {
170 try {
171 if (null != stmt) {
172 stmt.close();
173 }
174 } catch (Exception ex) {
175 LOG.warn("Exception while closing connection/stmt", ex);
176 }
177 }
178 }
179
180 @After
181 public void tearDown() {
182 super.tearDown();
183 try {
184 manager.close();
185 } catch (SQLException sqlE) {
186 LoggingUtils.logAll(LOG, "Got SQLException: ", sqlE);
187 }
188 }
189
190 @Test
191 public void testDb2Import() throws IOException {
192
193 runDb2Test(ExpectedResults);
194
195 }
196
197 private String [] getArgv() {
198 ArrayList<String> args = new ArrayList<String>();
199
200 CommonArgs.addHadoopFlags(args);
201 args.add("--connect");
202 args.add(CONNECT_STRING);
203 args.add("--username");
204 args.add(DATABASE_USER);
205 args.add("--password");
206 args.add(DATABASE_PASSWORD);
207 args.add("--warehouse-dir");
208 args.add(getWarehouseDir());
209
210 args.add("--m");
211 args.add("1");
212
213 for (String arg : extraArgs) {
214 args.add(arg);
215 }
216
217 return args.toArray(new String[0]);
218 }
219
220 private void runDb2Test(String expectedResults) throws IOException {
221
222 Path warehousePath = new Path(this.getWarehouseDir());
223 Path tablePath = new Path(warehousePath, getTableName());
224 Path filePath = new Path(tablePath, "part-m-00000");
225
226 File tableFile = new File(getTableName().toString());
227 if (tableFile.exists() && tableFile.isDirectory()) {
228 // remove the directory before running the import.
229 FileListing.recursiveDeleteDir(tableFile);
230 }
231
232 String [] argv = getArgv();
233 try {
234 runImportAll(argv);
235 } catch (IOException ioe) {
236 LOG.error("Got IOException during import: " + ioe.getMessage());
237 }
238
239 File f = new File(filePath.toString());
240 assertTrue("Could not find imported data file", f.exists());
241 BufferedReader r = null;
242 try {
243 // Read through the file and make sure it's all there.
244 r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
245 assertEquals(expectedResults, r.readLine());
246 } catch (IOException ioe) {
247 LOG.error("Got IOException verifying results: " + ioe.getMessage());
248 } finally {
249 IOUtils.closeStream(r);
250 }
251 }
252
253 private void runImportAll(SqoopTool tool,String [] argv) throws IOException {
254 // run the tool through the normal entry-point.
255 int ret;
256 try {
257 Configuration conf = getConf();
258 SqoopOptions opts = getSqoopOptions(conf);
259 Sqoop sqoop = new Sqoop(tool, conf, opts);
260 ret = Sqoop.runSqoop(sqoop, argv);
261 //ret = tool.run(opts);
262 } catch (Exception e) {
263 LOG.error("Got exception running Sqoop: " + e.toString());
264 ret = 1;
265 }
266
267 // expect a successful return.
268 if (0 != ret) {
269 throw new IOException("Failure during job; return status " + ret);
270 }
271 }
272
273 /** run an import using the default ImportTool. */
274 protected void runImportAll(String [] argv) throws IOException {
275 runImportAll(new ImportAllTablesTool(), argv);
276 }
277 }